BEAM ME UP SCOTTY

Co-Authored-By: Liza Tsibur <liza@activecountermeasures.com>

.dockerignore

@@ -0,0 +1,2 @@
+**/logs*
+test_data

.env

@@ -0,0 +1,41 @@
+# Stores values used during the build process
+# THIS FILE IS COMMITTED TO THE REPOSITORY
+# Any secrets or sensitive values should be stored in a .env.development or .env.local file, as these are not committed
+CLICKHOUSE_VERSION=24.1.6
+BUILDPLATFORM=linux/arm64
+LOGS=./test_data
+# TODO change this from loggies 
+APP_LOGS=./loggies
+CONFIG_DIR=./deployment
+CONFIG_FILE=./config.hjson
+SYSLOG_ADDRESS=localhost:514
+DB_ADDRESS=localhost:9000
+
+# docker compose build --env-file .env.production
+# cp ./.env.production /opt/rita/.env
+
+
+# ---
+# - name: Example playbook for Ubuntu
+#   hosts: default
+#   become: true
+#   tasks:
+#     - name: Create a directory if it does not exist
+#       ansible.builtin.file:
+#         path: /etc/rita
+#         state: directory
+#         mode: '0644'
+#     - name: Another symbolic mode example, adding some permissions and removing others
+#       ansible.builtin.copy:
+#         src: ~/repos/rita-v2/.env.production
+#         dest: /etc/rita/.env
+#         owner: root
+#         group: root
+#         mode: u+rw,g-wx,o-rwx
+#     - name: Another symbolic mode example, adding some permissions and removing others
+#       ansible.builtin.copy:
+#         src: ~/repos/rita-v2/deployment/*
+#         dest: /etc/rita
+#         owner: root
+#         group: root
+#         mode: u+rw,g-wx,o-rwx

.env.production

@@ -0,0 +1,11 @@
+# Stores values used during the build process
+# THIS FILE IS COMMITTED TO THE REPOSITORY
+# Any secrets or sensitive values should be stored in a .env.development or .env.local file, as these are not committed
+CLICKHOUSE_VERSION=24.1.6
+# BUILDPLATFORM=linux/arm64
+LOGS=/opt/zeek/logs
+CONFIG_DIR=/etc/rita
+CONFIG_FILE=/etc/rita/config.hjson
+SYSLOG_ADDRESS=syslogng:5514
+APP_LOGS=/var/log/rita
+DB_ADDRESS=db:9000

.github/CODEOWNERS

@@ -0,0 +1 @@
+@activecm/lumberghs

.github/workflows/build.yml

@@ -0,0 +1,57 @@
+name: Build Docker Images
+
+# on: [pull_request,workflow_dispatch]
+  
+  # pull_request: # TODO: delete this line
+  # TODO: comment this back in
+on:
+  push: 
+    branches: 
+      - "main"
+    tags:
+      - "v*.*.*"
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/activecm/rita-v2            
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{raw}}
+            type=sha        
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          # TODO: comment this back in
+#           push: ${{ github.event_name != 'pull_request' }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+

.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Generate Installer
+
+on:
+  release:
+    types:
+      - published
+      #- unpublished
+      - created
+      - edited
+      #- deleted
+      - prereleased
+      - released
+
+  workflow_run:
+    workflows: ["Build Docker Images"]
+    branches: [main]
+    types: 
+        - completed
+
+jobs:
+  upload:
+    name: Upload Artifacts
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - run: installer/generate_installer.sh
+      - uses: softprops/action-gh-release@v2
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+            files: rita-${{ github.ref_name }}.tar.gz

.github/workflows/test.yml

@@ -0,0 +1,48 @@
+name: Tests
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: 
+      labels: ubuntu-latest-m
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+      - name: Install dependencies
+        run: go get .
+      - name: Build
+        run: go build -v ./...
+      - name: Install tparse
+        run: go install github.com/mfridman/tparse@latest
+      - name: Unit Tests
+        run: go test --cover $(go list ./... | grep -v /integration | grep -v /database | grep -v /cmd | grep -v /viewer ) -json 2>&1 | tee unit.out && tparse -all --file=unit.out
+      - name: Integration Tests
+        shell: 'script -q -e -c "bash {0}"'
+        run: go test ./integration/... -json -timeout 1800s 2>&1 | tee integration.out && tparse -all --file=integration.out
+      - name: Database Tests
+        shell: 'script -q -e -c "bash {0}"'
+        run: go test ./database/... -json -timeout 1800s 2>&1 | tee database.out && tparse -all --file=database.out
+      - name: Cmd Tests
+        shell: 'script -q -e -c "bash {0}"'
+        run: go test ./cmd/... -json -timeout 1800s 2>&1 | tee cmd.out && tparse -all --file=cmd.out
+      - name: Viewer Tests
+        shell: 'script -q -e -c "bash {0}"'
+        run: go test ./viewer/... -json -timeout 1800s 2>&1 | tee viewer.out && tparse -all --file=viewer.out
+      - name: Upload test output on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: media
+          path: |
+            integration.out
+            unit.out
+            database.out
+            cmd.out
+            viewer.out
+      

.gitignore

@@ -0,0 +1,52 @@
+.DS_Store
+**/venv
+**/.venv
+*.log
+*.log.gz
+/logs/*
+/logs_*/*
+/venv/*
+/rita
+/rita-
+/*.out
+/*.test
+/*.zip
+/*.tar/
+
+**/.env*
+**/.terraform*
+**/*.tfstate*
+/installer/stage
+/installer/rita-*.tar.gz
+/installer/rita-*
+!/installer/*.md
+
+# only commit .env, .env.production, and test.env files
+!/.env
+!*/test.env
+!/.env.production
+
+# defined test logs
+!/test_data/dns_only/*[.log.gz]
+!/test_data/has_unknown_field/*[.log.gz]
+!/test_data/json_with_all_fields/*[.log.gz]
+!/test_data/open_conns/open/*[.log.gz]
+!/test_data/open_conns/closed/*[.log.gz]
+!/test_data/text_file/*[.log.gz]
+!/test_data/truncated/*[.log.gz]
+!/test_data/valid_json/*[.log.gz]
+!/test_data/valid_tsv/*[.log.gz]
+!/test_data/proxy/*[.log.gz]
+!/test_data/proxy_rolling/*[.log.gz]
+!/test_data/valid_tsv/*[.log.gz]
+!/test_data/dnscat2-ja3-strobe-agent/*[.log.gz]
+!/test_data/missing_host/*/*[.log.gz]
+!/test_data/open_sni/*[.log.gz]
+
+
+
+
+
+
+
+

.golangci.yml

@@ -0,0 +1,228 @@
+linters-settings:
+  # Checks for unchecked errors
+  errcheck:
+    # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. # Default: false
+    check-type-assertions: true
+
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`. # Default: false
+    check-blank: true # Default: false
+
+  # Provides diagnostics that check for bugs, performance and style issues
+  gocritic:
+    # Enable multiple checks by tags in addition to default checks.
+    # See https://github.com/go-critic/go-critic#usage -> section "Tags".
+    # Default: []
+    enabled-tags:
+      - diagnostic
+      - performance
+      - style
+    disabled-tags:
+      - experimental
+      - opinionated
+
+    # Settings passed to gocritic.
+    # The settings key is the name of a supported gocritic checker.
+    # The list of supported checkers can be find in https://go-critic.github.io/overview.
+    settings:
+      captLocal:
+        # Whether to restrict checker to params only
+        paramsOnly: false # Default: true
+      underef:
+        # Whether to skip (*x).method() calls where x is a pointer receiver
+        skipRecvDeref: false # Default: true
+      rangeExprCopy:
+        sizeThreshold: 512 # Default: 512
+        skipTestFuncs: true
+      rangeValCopy:
+        sizeThreshold: 1000  # Default: 128
+        skipTestFuncs: true
+      hugeParam:
+        sizeThreshold: 80 # Default: 80
+
+  # Examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes.
+  govet:
+    enable:
+      # Check for shadowed variables
+      - shadow
+    settings:
+      shadow:
+        # Whether to be strict about shadowing; can be noisy.
+        strict: false # Default: true
+
+  # Checks that functions with naked returns are not longer than a maximum size (can be zero).
+  nakedret:
+    max-func-lines: 30 # Default: 30
+
+  # analyzer that detects using os.Setenv instead of t.Setenv since Go1.17.
+  tenv:
+    # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures.
+    # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked.
+    # Default: false
+    all: false
+
+  # Reports ill-formed or insufficient nolint directives
+  nolintlint:
+    # Disable to ensure that all nolint directives actually have an effect.
+    # Default: false
+    allow-unused: true
+    # Exclude following linters from requiring an explanation.
+    # Default: []
+    allow-no-explanation: []
+    # Enable to require an explanation of nonzero length after each nolint directive.
+    # Default: false
+    require-explanation: true
+    # Enable to require nolint directives to mention the specific linter being suppressed.
+    # Default: false
+    require-specific: true
+  
+  revive:
+    rules:
+      - name: var-naming
+        disabled: true
+  
+  goconst:
+    # Minimum occurrences of constant string count to trigger issue.
+    # Default: 3
+    min-occurrences: 5
+    # Ignore test files.
+    # Default: false
+    ignore-tests: true
+  
+  stylecheck:
+    checks: ["all", "-ST1003"]
+
+linters:
+  # Set to true runs only fast linters
+  fast: false
+
+  enable:
+    # Check for pass []any as any in variadic func(...any)
+    - asasalint
+
+    # Checks for unusual ASCII identifiers.
+    - asciicheck
+
+    # Checks for dangerous unicode character sequences.
+    - bidichk
+
+    # Checks whether HTTP response body is closed successfully.
+    - bodyclose
+
+    # Check for two durations multiplied together.
+    - durationcheck
+
+    # Forces to not skip error check.
+    - errcheck 
+
+    # Checks `Err-` prefix for var and `-Error` suffix for error type.
+    - errname
+
+    # Suggests to use `%w` for error-wrapping.
+    - errorlint
+
+    # Checks for pointers to enclosing loop variables.
+    - exportloopref
+
+    # Validates go compiler directive comments (//go:)
+    - gocheckcompilerdirectives
+
+    # Finds repeated strings that could be replaced by a constant
+    - goconst
+
+    # Maintains checks which are currently not implemented in other linters.
+    - gocritic
+
+    # Checks standard formatting (whitespace, indentation, etc.)
+    - gofmt
+
+    # Checks 
+    - goimports
+
+    # Allow or ban replace directives in go.mod or force explanation for retract directives.
+    - gomoddirectives
+
+    # Powerful security-oriented linter. But requires some time to
+    # configure it properly, see https://github.com/securego/gosec#available-rules
+    - gosec
+
+    # Linter that specializes in simplifying code.
+    - gosimple
+
+    # Examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes.
+    - govet
+
+    # Detects when assignments to existing variables are not used
+    - ineffassign
+
+    # Spell checker
+    - misspell
+
+    # Checks that functions with naked returns are not longer than a maximum size (can be zero).
+    - nakedret
+
+    # Both require a bit more explicit returns.
+    - nilerr
+    - nilnil
+
+    # Finds sending HTTP request without context.Context.
+    - noctx
+
+    # Forces comment why another check is disabled.
+    - nolintlint
+
+    # Finds shadowing of Go's predeclared identifiers
+    - predeclared
+
+    # Lint your Prometheus metrics name
+    - promlinter
+
+    # Checks that package variables are not reassigned.
+    - reassign
+
+    # Drop-in replacement of `golint`.
+    - revive
+
+    # For `database/sql` package.
+    - rowserrcheck
+    - sqlclosecheck
+
+    # Detects bugs, suggests code simplifications, points out dead code, etc
+    - staticcheck
+
+    # Replacement for `golint`, similar to `revive`.
+    - stylecheck
+
+    # Test-related checks
+    - tenv
+    - testableexamples
+    - testifylint
+    - thelper
+    - tparallel
+
+    # Parses and type-checks
+    - typecheck
+
+    # Remove unnecessary type conversions
+    - unconvert
+
+    # Finds unused parameters
+    # - unparam
+
+    # Finds unused declarations
+    - unused
+
+    # Detect the possibility to use variables/constants from stdlib.
+    - usestdlibvars
+
+    # Finds wasted assignment statements.
+    - wastedassign
+
+    # detects the wrong usage of zerolog that a user forgets to dispatch zerolog.Event
+    - zerologlint
+
+issues:
+  exclude:
+    - "var-naming: don't use an underscore in package name"
+  exclude-rules:
+    - text: 'shadow: declaration of "(err|ctx)" shadows declaration at'
+      linters: [ govet ]

.vscode/rita.code-workspace

@@ -0,0 +1,55 @@
+{
+    "folders": [
+        {
+            "path": ".."
+        }
+    ],
+    "extensions": {
+        "recommendations": [
+            "qufiwefefwoyn.inline-sql-syntax",
+            "wayou.vscode-todo-highlight",
+            "golang.go",
+            "hjson.hjson",
+            "Tanh.hjson-formatter",
+        ]
+    },
+    "settings": {
+        "go.languageServerExperimentalFeatures": {
+            "diagnostics": false
+        },
+        "go.vetOnSave": "off",
+        "go.lintTool": "golangci-lint",
+        "go.lintFlags": [
+            "--fast",
+            "--config=${workspaceFolder}/.golangci.yml"
+        ],
+        "gopls": {
+            "ui.semanticTokens": true
+        },
+        "[go]": {
+            "editor.defaultFormatter": "golang.go",
+            "editor.formatOnSave": true
+        },
+        "[json]": {
+            "editor.defaultFormatter": "vscode.json-language-features",
+            "editor.formatOnSave": true
+        },
+        "[jsonc]": {
+            "editor.defaultFormatter": "vscode.json-language-features",
+            "editor.formatOnSave": true
+        },
+        "[hjson]": {
+            "editor.defaultFormatter": "Tanh.hjson-formatter",
+            "editor.formatOnSave": true
+        },
+        "hjson-formatter.options": {
+            "separator": true,
+            "condense": 100,
+            "bracesSameLine": true,
+            "emitRootBraces": true,
+            "quotes": "strings",
+            "space": 4,
+            "eol": "auto",
+        },
+    }
+}

CONTRIBUTING.md

@@ -0,0 +1,156 @@
+# Contribution Guidelines
+
+Thank you for considering contributing to RITA! We appreciate your interest and support. Please take a moment to review the following guidelines before making your contribution.
+
+## Table of Contents
+1. [Code of Conduct](#code-of-conduct)
+2. [How to Contribute](#how-to-contribute)
+    - [Reporting Bugs](#reporting-bugs)
+    - [Suggesting Features](#suggesting-features)
+    - [Answering Questions](#answering-questions)
+    - [Contributing Code](#contributing-code)
+3. [Development Process](#development-process)
+    - [Prerequisites](#prerequisites)
+    - [Setting Up Your Development Environment](#setting-up-your-development-environment)
+    - [Style](#style)
+    - [Testing](#testing)
+    - [Submitting a Pull Request](#submitting-a-pull-request)
+
+## Code of Conduct
+
+We strive to maintain a welcoming and inclusive community for everyone who wants to contribute to this project. Therefore, we kindly ask you to adhere to the following code of conduct in all interactions related to this project:
+
+* Treat all contributors with respect and consideration. Discrimination, harassment, or offensive behavior will not be tolerated.
+* Communicate in a productive and professional manner.
+* Be capable of both giving and accepting constructive feedback
+* Work with others to resolve conflicts and reach a consensus.
+* Value and acknowledge the contributions of others.
+* Use welcoming and inclusive language
+* Help create a positive and collaborative environment by treating everyone with kindness and empathy
+
+
+## How to Contribute
+
+### Reporting Bugs
+
+If you find a bug, please first verify that the issue has not already been reported [here](https://github.com/activecm/rita/issues). If it has not been reported yet, open a new issue. Include as much detail as possible to help us understand and reproduce the bug. Follow these guidelines:
+
+1. Use a clear and descriptive title.
+2. Provide a step-by-step description of the bug and how to reproduce it.
+3. Include any relevant screenshots, logs, or error messages.
+4. Mention the version of the project you are using and the environment (e.g., OS, CPU, RAM, hard drive space, etc.)
+
+### Suggesting Features
+
+To suggest a new feature, please create an issue with the following information:
+
+1. Use a clear and descriptive title.
+2. Provide a detailed description of the feature and its benefits.
+3. Explain any potential use cases or examples of how the feature could be used.
+4. If possible, provide examples or mockups of the feature.
+
+### Answering Questions
+
+Rather than creating an issue to request help with the project, we encourage users to join our [Threat Hunter Discord](https://discord.gg/threathunter) to ask questions and seek guidance. The **#rita** and **#rita-development** channels are focused on this project. If you have experience with the project, please consider helping others by answering their questions on Discord. This will help us keep the issue tracker focused on bugs and feature requests and provide a way for our community to communicate and support each other.
+
+### Contributing Code
+
+The steps below outline the process for contributing code to this project. Before you begin, please ensure that you have read and understood our [Code of Conduct](#code-of-conduct). Please also review the [Development Process](#development-process) section for more detailed guidelines on this process.
+
+1. Set up your development environment as described in the [Prerequisites](#prerequisites) section.
+3. Verify that your intended contribution is associated with an issue. If not, create one using the guidelines above.
+2. Create a new branch for your contribution (e.g., `feature/awesome-feature` or `bugfix/issue`).
+3. Make your changes
+4. Write tests to cover your changes.
+5. Commit your changes with clear and descriptive commit messages.
+6. Push your branch to your forked repository.
+7. Create a pull request to the `main` branch of the original repository.
+8. Your pull request will be reviewed by the maintainers, and you may be asked to make changes before it is accepted.
+
+## Development Process
+
+### Prerequisites
+Contribution to this project will require the following:
+ * [Go](https://golang.org/doc/install) (Check `go.mod` for the correct version to install)
+ * [Docker](https://docs.docker.com/engine/installation/)
+
+### Setting Up Your Development Environment
+1. Fork the repository on GitHub.
+2. Clone your fork to your local machine:
+   ```
+   git clone https://github.com/your-username/rita.git 
+   ```
+3. Navigate to the project directory:
+   ```
+   cd rita
+   ```
+3. Install the project dependencies:
+   ```
+   go mod download
+   ```
+4. Run the tests to verify that everything is working correctly:
+   ```
+   go test ./...
+   ```
+5. Run the project:
+    * Using Go:
+        ```
+        docker compose up -d
+        go run main.go <command> <flags>
+        ```
+    * Using a Compiled Binary:
+        ```
+        make
+        ./rita <command> <flags>
+        ```
+    * Using Docker:
+        ```
+        ./rita.sh
+        ```
+5. Make your changes and ensure that they pass the tests before submitting a pull request.
+
+For more information about setting up a development environment, see the [docs](docs/Development.md).
+
+### Style
+Please follow these guidelines to ensure consistency across the project:
+
+- Use clear and descriptive variable and function names.
+- Write clear and concise comments where necessary.
+- Format your code using a golang extension on your editor or the `gofmt` tool.
+   ```
+   gofmt -s -w .
+   ```
+- Use golangci-lint for linting with the configuration specified in .golangci.yml:
+   ```
+   golangci-lint run
+   ```
+- Write tests for new functionality and ensure existing tests pass.
+
+### Testing
+Writing tests is a crucial part of contributing to our project. Please ensure that your code changes include tests where applicable. Here are some guidelines for writing and running tests:
+
+**Unit Tests**: 
+Write unit tests for individual functions and methods. Place your test files in the same package as the code being tested and name the test files with a _test.go suffix.
+
+**Integration Tests**:
+Write integration tests for testing interactions between different components. Integration tests should also be written to test the final results of the import process against known, expected values. These tests should be placed in the integration package.
+
+
+### Submitting a Pull Request
+When submitting a pull request, please follow these steps:
+- Verify that your PR is based on a single logical change.
+- Check that your commit messages are clear and descriptive.
+- Ensure that your code follows our [style guide](#style-guide)
+- Write or update tests to cover your changes.
+- Verify that all tests pass.
+- Provide a clear and concise summary of all changes in the PR description.
+- Reference all relevant issues in the PR description (e.g., "Closes #123").
+
+Once you have submitted your PR, the maintainers will review it and provide feedback. You may be asked to make changes before your PR is accepted.
+
+### PR Has Been Reviewed and Merged
+🎉✨ Congratulations and thank you for contributing to RITA! ✨🎉
+
+
+## Contact
+If you have any questions please reach out to us on our [Threat Hunter Discord](https://discord.gg/threathunter) via the **#rita** or **#rita-development** channels.

Dockerfile

@@ -0,0 +1,43 @@
+FROM --platform=$BUILDPLATFORM golang:1.22-alpine as rita-builder
+
+ARG TARGETOS
+ARG TARGETARCH
+
+# install dependencies
+RUN apk add --no-cache git make ca-certificates wget build-base
+
+# set the working directory
+WORKDIR /go/src/github.com/activecm/rita
+
+# cache dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# copy the rest of the code
+COPY . ./
+
+# Change ARGs with --build-arg to target other architectures
+# Produce a self-contained statically linked binary
+ARG CGO_ENABLED=0
+
+# Set the build target architecture and OS
+ARG GOARCH=${TARGETARCH}
+ARG GOOS=${TARGETOS}
+# Passing arguments in to make result in them being set as
+# environment variables for the call to go build
+RUN make CGO_ENABLED=$CGO_ENABLED GOARCH=$GOARCH GOOS=$GOOS
+# RUN mkdir /var/log/rita
+# RUN chmod 0755 /var/log/rita
+
+FROM alpine
+
+# RUN mkdir /logs
+# RUN chmod 0755 /logs
+
+WORKDIR /
+# COPY --from=rita-builder /go/src/github.com/activecm/rita/config.hjson /etc/rita/config.hjson
+COPY --from=rita-builder /go/src/github.com/activecm/rita/rita /rita
+# COPY --from=rita-builder /go/src/github.com/activecm/rita/.env.production /.env
+# COPY --from=rita-builder /go/src/github.com/activecm/rita/deployment /etc/rita
+
+ENTRYPOINT ["/rita"]

LICENSE

@@ -1,7 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -620,55 +620,3 @@ copy of the Program in return for a fee.
 
                      END OF TERMS AND CONDITIONS
 
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
-    <program>  Copyright (C) <year>  <name of author>
-    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<https://www.gnu.org/licenses/>.
-
-  The GNU General Public License does not permit incorporating your program
-into proprietary programs.  If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.  But first, please read
-<https://www.gnu.org/licenses/why-not-lgpl.html>.

Makefile

@@ -0,0 +1,10 @@
+VERSION := $(shell git describe --always --abbrev=0 --tags)
+
+LDFLAGS := -ldflags='-X main.Version=$(VERSION) -extldflags "-static"'
+CGO_ENABLED ?= 0
+GOARCH ?= $(shell go env GOARCH)
+GOOS ?= $(shell go env GOOS)
+
+.PHONY: rita
+rita:
+	CGO_ENABLED=$(CGO_ENABLED) GOARCH=$(GOARCH) GOOS=$(GOOS) go build $(LDFLAGS) -o rita

README.md

@@ -0,0 +1,127 @@
+# RITA (Real Intelligence Threat Analytics)
+
+[![RITA Logo](rita-logo.png)](https://www.activecountermeasures.com/free-tools/rita/)
+
+If you get value out of RITA and would like to go a step further with hunting automation, futuristic visualizations, and data enrichment, then take a look at [AC-Hunter](https://www.activecountermeasures.com/).
+
+Sponsored by [Active Countermeasures](https://activecountermeasures.com/).
+
+---
+
+RITA is an open source framework for network traffic analysis.
+
+The framework ingests [Zeek Logs](https://www.zeek.org/) in TSV or JSON format, and currently supports the following major features:
+ - **Beaconing Detection**: Search for signs of beaconing behavior in and out of your network
+- **Long Connection Detection**: Easily see connections that have communicated for long periods of time
+ - **DNS Tunneling Detection**: Search for signs of DNS based covert channels
+ - **Threat Intel Feed Checking**: Query threat intel feeds to search for suspicious domains and hosts
+
+ ## Quick Start
+ Please see our recommended [System Requirements](docs/System%20Requirements.md).
+
+1. Download the [RITA Installer](https://github.com/activecm/rita/releases) for the desired version.
+
+2. Uncompress the installer tarfile.
+   ```
+   tar -xf rita-<version>-installer.tar.gz
+   ```
+3. Run the install script. 
+   ```
+   ./rita-<version>-installer/install_rita.sh <hosts to install on>
+   ```
+   To install RITA on the local system, pass `localhost` to the installer.
+   To install RITA on one or more remote systems, pass a comma separated list of IPs or `user@ip` or FQDNs to the installer.
+   For example:
+   ```
+   ./rita-<version>-installer/install_rita.sh "root@4.4.4.4,8.8.8.8,mydomain.com"
+   ```
+
+
+### Supported Platforms
+- ✅: Official Support
+- ⚠️: Unofficial Support
+- ❌: Unsupported
+
+| OS              | Versions | Platform | Status | 
+| :---------------- | :------ | :---- | :----: | 
+| CentOS         | `9 Stream` | `amd64` | ✅ |
+| Rocky         | `9` | `amd64` | ✅ |
+| Ubuntu        |   `24.04`   | `amd64`| ✅ |
+| Windows        |    | | ❌ |
+<!-- TODO: eventually add support -->
+<!-- | MacOS        |   `Sonoma`   | `intel\|arm` | ✅ | -->
+
+## Installing Zeek
+If you do not already have Zeek installed, it can be installed from [docker-zeek](https://github.com/activecm/docker-zeek).
+
+```
+sudo wget -O /usr/local/bin/zeek https://raw.githubusercontent.com/activecm/docker-zeek/master/zeek
+
+sudo chmod +x /usr/local/bin/zeek
+
+zeek start
+```
+
+## Importing
+Import data into RITA using the `import` command:
+```
+rita import --database=mydatabase --logs=~/mylogs
+```
+
+`database` is what you would like to name the dataset
+
+`logs` is the path to the Zeek logs you wish to import
+
+For datasets that should accumulate data over time, such as importing new logs from the current Zeek sensor on a cron job, use the `--rolling` flag during creation and each subsequent import into the dataset.
+
+To destroy and recreate a dataset, use the `--rebuild` flag.
+
+## Configuration
+See [Configuration](/docs/Configuration.md) for details on adjusting scoring.
+
+## Searching
+
+RITA follows a GitHub-style search syntax. Each field follows the `<field>:<value>` format, with each search criteria separated by a space. 
+For example: 
+```
+src:192.168.88.2 dst:165.227.88.15 beacon:>=90 sort:duration-desc
+```
+
+### Supported Search Fields
+
+| Column              | Field | Operators | Data Type |
+| :---------------- | :------ | :---- | :---- |
+| Severity        |   `severity`   |  | `critical\|high\|medium\|low` |
+| Source           |   `src`   |  | IP address |
+| Destination           |   `dst`   |  | IP address, FQDN |
+| Beacon Score           |   `beacon`   | `>, >=, <, <=` | whole number
+| Duration    |  `duration`   | `>, >=, <, <=` | string, ex:(`2h45m`)
+| Subdomains |  `subdomains`   | `>, >=, <, <=` | whole number |
+| Threat Intel |  `threat_intel`   | | `true\|false` |
+
+### Supported Sort Fields
+The sort syntax is `sort:<column>-<sort direction>`, with the sort direction being `asc` for ascending or `desc` for descending.
+
+Supported Columns:
+
+- severity
+- beacon
+- duration
+- subdomains
+
+## CSV Output
+To output the results to CSV instead of viewing them within the terminal UI, pass the `--stdout` or `-o` flag to the `view` command:
+
+*The flag must be before the name of the dataset.*
+```
+rita view --stdout mydataset
+```
+
+## Terminal UI Color Support
+The terminal UI (TUI) supports colorful output by default. It does not need to be enabled. 
+
+Check the value of the `"$TERM"` variable, this should be `xterm-256color`. If it is not, please set this variable in your OS's version of a `~/.bash_profile`, `~/.profile`, etc.
+
+Depending on the color theme of your terminal, the TUI will adjust to either a light mode or a dark mode.
+
+If you're really fancy and like pretty colors, consider using the [Catpuccin](https://catppuccin.com/ports?q=terminal) theme!

analysis/analysis.go

@@ -0,0 +1,363 @@
+package analysis
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/logger"
+	"activecm/rita/util"
+	"context"
+	"fmt"
+	"math"
+	"net"
+	"runtime"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
+)
+
+func (r ThreatMixtape) FilterValue() string { return r.Src.String() }
+func (i ThreatMixtape) Title() string       { return i.Dst.String() }
+func (i ThreatMixtape) Description() string { return fmt.Sprint(i.FinalScore * 100) }
+
+type Analyzer struct {
+	Database        *database.DB
+	ImportID        util.FixedString
+	Config          *config.Config
+	AnalysisWorkers int
+	WriterWorkers   int
+	UconnChan       chan AnalysisResult
+	maxTS           time.Time
+	minTS           time.Time
+	maxTSBeacon     time.Time
+	minTSBeacon     time.Time
+	networkSize     uint64
+	useCurrentTime  bool
+	skipBeaconing   bool
+	firstSeenMaxTS  time.Time
+
+	writer *database.BulkWriter
+}
+
+type ThreatMixtape struct {
+	AnalyzedAt time.Time        `ch:"analyzed_at"`
+	ImportID   util.FixedString `ch:"import_id"`
+
+	// Base connection details
+	AnalysisResult
+
+	FinalScore float32 `ch:"final_score"`
+	// BEACONS
+	Beacon
+	BeaconThreatScore float32 `ch:"beacon_threat_score"` // bucketed beacon score
+	BeaconType        string  `ch:"beacon_type"`
+
+	//  LONG CONNECTIONS
+	LongConnScore float32 `ch:"long_conn_score"`
+
+	// Strobe
+	Strobe      bool    `ch:"strobe"`
+	StrobeScore float32 `ch:"strobe_score"`
+
+	// C2 over DNS
+	C2OverDNSScore           float32 `ch:"c2_over_dns_score"`
+	C2OverDNSDirectConnScore float32 `ch:"c2_over_dns_direct_conn_score"`
+
+	// Threat Intel
+	ThreatIntel      bool    `ch:"threat_intel"`
+	ThreatIntelScore float32 `ch:"threat_intel_score"`
+
+	// **** MODIFIERS ****
+	// for modifiers detected during the modifiers phase
+	ModifierName  string  `ch:"modifier_name"`
+	ModifierScore float32 `ch:"modifier_score"`
+	ModifierValue string  `ch:"modifier_value"`
+
+	// modifiers that are able to be added to the same row as the threat indicator scores
+	// these are detected during the analysis phase (in the spagooper)
+	PrevalenceScore          float32 `ch:"prevalence_score"`
+	FirstSeenScore           float32 `ch:"first_seen_score"`
+	ThreatIntelDataSizeScore float32 `ch:"threat_intel_data_size_score"`
+	MissingHostHeaderScore   float32 `ch:"missing_host_header_score"`
+
+	// FirstSeenMod           float32 `ch:"first_seen_mod"`
+	// PrevalenceMod          float32 `ch:"prevalence_mod"`
+	// MissingHostHeaderMod   float32 `ch:"missing_host_header_mod"`
+	// C2OverDNSDirectConnMod float32 `ch:"c2_over_dns_direct_conn_mod"`
+	// MIMETypeMismatchMod    float32 `ch:"mime_type_mismatch_mod"`
+	// RareSignatureMod       float32 `ch:"rare_signature_mod"`
+}
+
+// NewAnalyzer returns a new Analyzer object
+func NewAnalyzer(db *database.DB, cfg *config.Config, importID util.FixedString, minTS, maxTS, minTSBeacon, maxTSBeacon time.Time, useCurrentTime bool, skipBeaconing bool) (*Analyzer, error) {
+
+	// create a rate limiter to control the rate of writing to the database
+	limiter := rate.NewLimiter(5, 5)
+	networkSize, err := db.GetNetworkSize(minTS) // use true min TS for network size
+	if err != nil {
+		return nil, err
+	}
+	var firstSeenMaxTS time.Time
+	if !useCurrentTime {
+		// _, firstSeenMaxTS, _, _, err = db.GetTrueMinMaxTimestamps()
+		// if err != nil {
+		// 	return nil, fmt.Errorf("could not get max timestamp for first seen analysis: %w", err)
+		// }
+		firstSeenMaxTS = maxTS
+	}
+
+	workers := int(math.Floor(math.Max(4, float64(runtime.NumCPU())/2)))
+	return &Analyzer{
+		Database:        db,
+		Config:          cfg,
+		ImportID:        importID,
+		AnalysisWorkers: workers,
+		WriterWorkers:   workers,
+		useCurrentTime:  useCurrentTime,
+		maxTS:           maxTS,
+		minTS:           minTS,
+		maxTSBeacon:     maxTSBeacon,
+		minTSBeacon:     minTSBeacon,
+		firstSeenMaxTS:  firstSeenMaxTS,
+		skipBeaconing:   skipBeaconing,
+		networkSize:     networkSize,
+		UconnChan:       make(chan AnalysisResult),
+		writer:          database.NewBulkWriter(db, cfg, workers, db.GetSelectedDB(), "threat_mixtape", "INSERT INTO {database:Identifier}.threat_mixtape", limiter, false),
+	}, nil
+}
+
+func (analyzer *Analyzer) Analyze() error {
+	logger := logger.GetLogger()
+
+	// log the start time of the analysis
+	start := time.Now()
+	logger.Debug().Msg("Starting Analysis")
+
+	// create an error group to manage the analysis threads
+	analysisErrGroup, ctx := errgroup.WithContext(context.Background())
+
+	// create analysis calculation workers
+	for i := 0; i < analyzer.AnalysisWorkers; i++ {
+		analysisErrGroup.Go(func() error {
+			err := analyzer.runAnalysis()
+			return err
+		})
+	}
+
+	// create analysis writer workers
+	for i := 0; i < analyzer.WriterWorkers; i++ {
+		analyzer.writer.Start(i)
+	}
+
+	// start spagooper to feed anlysis threads
+	err := analyzer.Spagoop(ctx)
+	if err != nil {
+		return fmt.Errorf("could not perform spagoop analysis: %w", err)
+	}
+
+	// wait for all analysis threads to finish
+	if err := analysisErrGroup.Wait(); err != nil {
+		logger.Fatal().Err(err).Msg("could not perform beacon analysis")
+		return err
+	}
+
+	// close the mixtape writer
+	analyzer.writer.Close()
+
+	// log the end time of the analysis
+	end := time.Now()
+	diff := time.Since(start)
+	logger.Info().Str("elapsed_time", diff.String()).Time("analysis_began", start).Time("analysis_finished", end).Msg("Finished Analysis! 🎉")
+
+	return nil
+}
+
+func (analyzer *Analyzer) runAnalysis() error {
+	logger := logger.GetLogger()
+
+	// loop over the uconn channel to process each entry
+	for entry := range analyzer.UconnChan {
+		// create a new mixtape entry to store the analysis results
+		mixtape := &ThreatMixtape{
+			AnalyzedAt:     analyzer.Database.ImportStartedAt.Truncate(time.Microsecond),
+			ImportID:       analyzer.ImportID,
+			AnalysisResult: entry,
+			BeaconType:     entry.BeaconType,
+		}
+
+		// set the first seen historical value
+		firstSeenHistorical, replaced := util.ValidateTimestamp(entry.FirstSeenHistorical)
+		if replaced {
+			logger.Debug().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("missing_host_count", fmt.Sprint(entry.MissingHostCount)).Str("fqdn", entry.FQDN).Msg("historical first seen timestamp was missing")
+		}
+
+		// if the last seen timestamp was not valid, then this entry cannot be inserted into the mixtape
+		// because modifiers require linking up with the last seen date
+		// this should log a warning as this is a bugs
+		lastSeen, replaced := util.ValidateTimestamp(entry.LastSeen)
+		if replaced {
+			logger.Debug().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("missing_host_count", fmt.Sprint(entry.MissingHostCount)).Str("fqdn", entry.FQDN).Msg("last seen timestamp was missing")
+		}
+
+		mixtape.FirstSeenHistorical = firstSeenHistorical
+		mixtape.LastSeen = lastSeen
+
+		hasThreatIndicator := false
+
+		// C2 OVER DNS
+		if entry.TLD != "" && entry.SubdomainCount > 0 {
+			// run c2 over dns analysis on entry if the TLD is a known c2 domain
+			c2OverDNSScore := calculateBucketedScore(float64(entry.SubdomainCount), analyzer.Config.Scoring.C2ScoreThresholds)
+
+			hash, err := util.NewFixedStringHash(entry.TLD)
+			if err != nil {
+				logger.Debug().Str("src", entry.Src.String()).Str("fqdn", entry.FQDN).Msg("could not create hash from TLD")
+			}
+			mixtape.Hash = hash
+			mixtape.FQDN = entry.TLD
+			if entry.SubdomainCount >= uint64(analyzer.Config.Scoring.C2SubdomainThreshold) {
+				hasThreatIndicator = true
+				mixtape.C2OverDNSScore = c2OverDNSScore
+				// run c2 over dns direct connection analysis
+				if shouldHaveC2OverDNSDirectConnModifier(entry.DirectConns, entry.QueriedBy) {
+					mixtape.C2OverDNSDirectConnScore = analyzer.Config.Modifiers.C2OverDNSDirectConnScoreIncrease
+				}
+			}
+
+		} else {
+
+			// ALL OTHER THREAT INDICATORS
+			// Run beaconing as long as there are min/max beacon timestamps
+			if !analyzer.skipBeaconing {
+				// run beacon analysis on entry if there are enough unique connections and the overall connection count is less than a strobe (1 connection per second)
+
+				if entry.TSUnique >= uint64(analyzer.Config.Scoring.Beacon.UniqueConnectionThreshold) && entry.Count < 86400 {
+					beacon, err := analyzer.analyzeBeacon(&entry)
+					if err != nil {
+						continue // all the errors will get logged in the beacon analyzer so we get a line number
+					}
+					beaconThreatScore := calculateBucketedScore(float64(beacon.Score*100), analyzer.Config.Scoring.Beacon.ScoreThresholds)
+					hasThreatIndicator = true
+					mixtape.Beacon = beacon
+					mixtape.BeaconThreatScore = beaconThreatScore
+				}
+			}
+
+			// run long connection analysis on entry if the total duration is greater than the minimum duration threshold
+			if entry.TotalDuration >= float64(analyzer.Config.Scoring.LongConnectionMinimumDuration) {
+				longConnScore := calculateBucketedScore(entry.TotalDuration, analyzer.Config.Scoring.LongConnectionScoreThresholds)
+				hasThreatIndicator = true
+				mixtape.LongConnScore = longConnScore
+			}
+
+			// record entry as a strobe if the overall connection count meets the strobe threshold (1 connection per second)
+			if entry.Count >= 86400 {
+				hasThreatIndicator = true
+				mixtape.Strobe = true
+				mixtape.StrobeScore = analyzer.Config.Scoring.StrobeImpact.Score
+			}
+
+			// MODIFIERS
+			// due to performance impact, these modifiers are scored here instead of in the modifier package
+			// MISSING HOST HEADER MODIFIER
+			if entry.MissingHostCount > 0 {
+				mixtape.MissingHostHeaderScore = analyzer.Config.Modifiers.MissingHostCountScoreIncrease
+			}
+
+			// Threat Intel Data Size Score
+			if entry.OnThreatIntel {
+				if entry.TotalBytes >= analyzer.Config.Modifiers.ThreatIntelDataSizeThreshold {
+					mixtape.ThreatIntelDataSizeScore = analyzer.Config.Modifiers.ThreatIntelScoreIncrease
+				}
+			}
+
+		}
+
+		if hasThreatIndicator {
+
+			// Modifiers that apply to all connection types
+			// first seen scoring
+			// use the current time to score against unless useCurrentTime is false
+			relativeTime := util.GetRelativeFirstSeenTimestamp(analyzer.useCurrentTime, analyzer.firstSeenMaxTS)
+			timeSince := relativeTime.Sub(entry.FirstSeenHistorical)
+			daysSinceFirstSeen := float32(timeSince.Hours() / 24)
+
+			if daysSinceFirstSeen <= analyzer.Config.Modifiers.FirstSeenIncreaseThreshold {
+				mixtape.FirstSeenScore = analyzer.Config.Modifiers.FirstSeenScoreIncrease
+			} else if daysSinceFirstSeen >= analyzer.Config.Modifiers.FirstSeenDecreaseThreshold {
+				mixtape.FirstSeenScore = -1 * analyzer.Config.Modifiers.FirstSeenScoreDecrease
+			}
+
+			// Prevalence Scoring
+			if entry.Prevalence <= analyzer.Config.Modifiers.PrevalenceIncreaseThreshold {
+				mixtape.PrevalenceScore = analyzer.Config.Modifiers.PrevalenceScoreIncrease
+			} else if entry.Prevalence >= analyzer.Config.Modifiers.PrevalenceDecreaseThreshold {
+				mixtape.PrevalenceScore = -1 * analyzer.Config.Modifiers.PrevalenceScoreDecrease
+			}
+
+			// record entry as a threat intel if the entry is marked as threat intel
+			if entry.OnThreatIntel {
+				mixtape.ThreatIntel = true
+				mixtape.ThreatIntelScore = analyzer.Config.Scoring.ThreatIntelImpact.Score
+			}
+
+			// check to see if any of the workers cancelled before sending another entry to the writer
+			analyzer.writer.WriteChannel <- mixtape
+		}
+	}
+
+	return nil
+}
+
+func calculateBucketedScore(value float64, thresholds config.ScoreThresholds) float32 {
+	base := float64(thresholds.Base)
+	low := float64(thresholds.Low)
+	medium := float64(thresholds.Med)
+	high := float64(thresholds.High)
+
+	// convert category scores to integers for calculation
+	noneScore := config.NONE_CATEGORY_SCORE * 100
+	lowScore := config.LOW_CATEGORY_SCORE * 100
+	mediumScore := config.MEDIUM_CATEGORY_SCORE * 100
+	highScore := config.HIGH_CATEGORY_SCORE * 100
+
+	score := float32(0)
+
+	// interpolate scores between the threat category bucket thresholds
+	switch {
+	// (Low)    1-4hrs
+	case value < base:
+		return 0
+	case value < low:
+		score = float32(noneScore + (value-base)/(low-base)*(lowScore-noneScore))
+	// (Medium) 4-8hrs
+	case value >= low && value < medium:
+		score = float32(lowScore + (value-low)/(medium-low)*(mediumScore-lowScore))
+	// (High)   8-12hrs+
+	case value >= medium:
+		// cap the maximum duration score value to the High category threshold because we're not scoring any higher than this
+		cappedValue := math.Min(value, high)
+		score = float32(mediumScore + (cappedValue-medium)/(high-medium)*(highScore-mediumScore))
+	}
+	return score / 100
+}
+
+// shouldHaveC2OverDNSDirectConnModifier returns true if no ips other than the ones in queriedby made connections to this domain
+func shouldHaveC2OverDNSDirectConnModifier(directConns, queriedBy []net.IP) bool {
+	if len(queriedBy) > 0 {
+		queried := make(map[string]struct{})
+		for _, ip := range queriedBy {
+			queried[ip.String()] = struct{}{}
+		}
+
+		// check for any ips in direct conns that aren't in queried by
+		for _, ip := range directConns {
+			if _, ok := queried[ip.String()]; !ok {
+				return false
+			}
+		}
+
+	}
+	// apply direct conn modifier if no ips other than the ones in queried by made connections to this domain
+	return true
+}

analysis/analysis_test.go

@@ -0,0 +1,102 @@
+package analysis
+
+import (
+	"activecm/rita/config"
+	"log"
+	"testing"
+
+	"github.com/joho/godotenv"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMain(m *testing.M) {
+	err := godotenv.Load("../.env")
+	if err != nil {
+		log.Fatal("Error loading .env file")
+	}
+	m.Run()
+}
+
+/* **** THREAT SCORE BUCKETS ****
+Each Threat Indicator's score is placed into a categorical severity bucket.
+Since the units of measurement for each threat indicator varies, we must
+place the scores into buckets based on predefined thresholds for each bucket.
+
+Each bucket has a set, non-configurable score that it applies to.
+None:   0%
+Low:    20%-40%
+Medium: 41%-60%
+High:   61%-80%
+*/
+
+func TestCalculateBucketedScore(t *testing.T) {
+	// verify that the score is greater than zero when the none threshold is zero
+	// (and the value is greater than zero)
+	score := calculateBucketedScore(1, config.ScoreThresholds{Base: 0, Low: 5, Med: 10, High: 15})
+	require.Greater(t, score, float32(0), "score must be greater than zero when the base threshold is zero & the value is greater than zero")
+
+	// verify that the score is 20% if the none threshold and the value are zero
+	// this allows configuration for any positive integer to score at least 20%
+	score = calculateBucketedScore(0, config.ScoreThresholds{Base: 0, Low: 5, Med: 10, High: 15})
+	require.InDelta(t, 0.2, score, 0.0001, "score must be 0.2 if the base threshold and the value are zero")
+
+	cfg, err := config.ReadFileConfig(afero.NewOsFs(), "../config.hjson")
+	require.NoError(t, err)
+
+	type testCase struct {
+		Name       string
+		Thresholds config.ScoreThresholds
+	}
+
+	testCases := []testCase{
+		{Name: "C2 Over DNS", Thresholds: cfg.Scoring.C2ScoreThresholds},
+		{Name: "Long Connections", Thresholds: cfg.Scoring.LongConnectionScoreThresholds},
+		{Name: "Beacons", Thresholds: cfg.Scoring.Beacon.ScoreThresholds},
+	}
+
+	for _, test := range testCases {
+		base := float64(test.Thresholds.Base)
+		low := float64(test.Thresholds.Low)
+		medium := float64(test.Thresholds.Med)
+		high := float64(test.Thresholds.High)
+
+		t.Run(test.Name, func(t *testing.T) {
+			score = calculateBucketedScore(base-1, test.Thresholds)
+			require.InDelta(t, 0, score, 0.00001, "score should be zero when value is lower than the None bucket threshold")
+
+			// verify score matches the low threshold bucket score
+			score = calculateBucketedScore(base, test.Thresholds)
+			require.InDelta(t, .20, score, 0.001, "score must match the base threshold bucket score")
+
+			score = calculateBucketedScore(low-1, test.Thresholds)
+			require.InDelta(t, .3995, score, 0.1, "score should be very close to the low threshold score if it is almost at the medium threshold")
+
+			score = calculateBucketedScore(low, test.Thresholds)
+			require.InDelta(t, .40, score, 0.001, "score must match the low threshold bucket score")
+
+			betweenLowAndMedium := low + ((medium - low) / 2)
+			score = calculateBucketedScore(betweenLowAndMedium, test.Thresholds)
+			require.InDelta(t, .50, score, 0.1, "score should interpolate between the low and medium bucket")
+
+			score = calculateBucketedScore(medium-1, test.Thresholds)
+			require.InDelta(t, .5995, score, 0.1, "score should be very close to the medium threshold score if it is almost at the high threshold")
+
+			score = calculateBucketedScore(medium, test.Thresholds)
+			require.InDelta(t, .60, score, 0.001, "score must match the medium threshold bucket score")
+
+			betweenMediumAndHigh := medium + ((high - medium) / 2)
+			score = calculateBucketedScore(betweenMediumAndHigh, test.Thresholds)
+			require.InDelta(t, .70, score, 0.1, "score should interpolate between the medium and high bucket")
+
+			score = calculateBucketedScore(high-1, test.Thresholds)
+			require.InDelta(t, .7995, score, 0.1, "score should be very close to the high threshold score if it is almost at the high threshold")
+
+			score = calculateBucketedScore(high, test.Thresholds)
+			require.InDelta(t, .80, score, 0.001, "score must match the high threshold bucket score")
+
+			score = calculateBucketedScore(high*2, test.Thresholds)
+			require.InDelta(t, .80, score, 0.001, "score must match the high threshold bucket score even if the value is larger than the high threshold")
+		})
+	}
+}

analysis/beacons.go

@@ -0,0 +1,788 @@
+package analysis
+
+import (
+	"activecm/rita/logger"
+	"activecm/rita/util"
+	"errors"
+	"fmt"
+	"math"
+	"slices"
+	"sort"
+
+	"github.com/montanaflynn/stats"
+)
+
+var ErrInvalidDatasetTimeRange = errors.New("invalid dataset timerange: min ts is greater than or equal to max ts")
+var ErrInputSliceEmpty = errors.New("input slice must not be empty")
+
+type Beacon struct {
+	BeaconType     string  `ch:"beacon_type"` // (sni, ip)
+	Score          float32 `ch:"beacon_score"`
+	TimestampScore float32 `ch:"ts_score"`
+	DataSizeScore  float32 `ch:"ds_score"`
+	HistogramScore float32 `ch:"hist_score"`
+	DurationScore  float32 `ch:"dur_score"`
+
+	TSIntervals      []int64 `ch:"ts_intervals"`
+	TSIntervalCounts []int64 `ch:"ts_interval_counts"`
+	DSSizes          []int64 `ch:"ds_sizes"`
+	DSCounts         []int64 `ch:"ds_size_counts"`
+}
+
+func (analyzer *Analyzer) analyzeBeacon(entry *AnalysisResult) (Beacon, error) {
+	logger := logger.GetLogger()
+	var beacon Beacon
+
+	// verify that minTSBeacon < maxTSBeacon
+	if analyzer.minTSBeacon.After(analyzer.maxTSBeacon) || analyzer.minTSBeacon.Equal(analyzer.maxTSBeacon) {
+		logger.Err(ErrInvalidDatasetTimeRange).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, ErrInvalidDatasetTimeRange
+	}
+
+	// calculate timestamp scores and metrics (unused fields are used by the test functions)
+	tsScore, _, _, intervals, intervalCounts, _, _, err := getTimestampScore(entry.TSList)
+	if err != nil {
+		logger.Err(err).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, err
+	}
+
+	// calculate data size scores and metrics
+	dsScore, _, _, dsSizes, dsCounts, _, _, err := getDataSizeScore(entry.BytesList)
+	if err != nil {
+		logger.Err(err).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, err
+	}
+
+	// calculate histogram score (note: we currently look at a 24 hour period)
+	_, _, totalBars, longestRun, histScore, err := getHistogramScore(analyzer.minTSBeacon.Unix(), analyzer.maxTSBeacon.Unix(), entry.TSList, analyzer.Config.Scoring.Beacon.HistModeSensitivity, analyzer.Config.Scoring.Beacon.HistBimodalOutlierRemoval, analyzer.Config.Scoring.Beacon.HistBimodalMinHours, 24)
+	if err != nil {
+		logger.Err(err).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, err
+	}
+
+	// calculate duration score
+	_, _, durScore, err := getDurationScore(analyzer.minTSBeacon.Unix(), analyzer.maxTSBeacon.Unix(), int64(entry.TSList[0]), int64(entry.TSList[len(entry.TSList)-1]), totalBars, longestRun, analyzer.Config.Scoring.Beacon.DurMinHours, analyzer.Config.Scoring.Beacon.DurIdealNumberOfConsistentHours)
+	if err != nil {
+		logger.Err(err).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, err
+	}
+
+	// calculate overall beacon score
+	score, err := getBeaconScore(tsScore, analyzer.Config.Scoring.Beacon.TsWeight,
+		dsScore, analyzer.Config.Scoring.Beacon.DsWeight,
+		durScore, analyzer.Config.Scoring.Beacon.DurWeight,
+		histScore, analyzer.Config.Scoring.Beacon.HistWeight)
+	if err != nil {
+		logger.Err(err).Caller().Str("src", entry.Src.String()).Str("dst", entry.Dst.String()).Str("fqdn", entry.FQDN).Send()
+		return beacon, err
+	}
+
+	// create beacon
+	// float64 values are cast to float32 for more efficient storage in the database, as the values
+	// are not expected to exceed the range of a float32. The cast is done here at the end of analysis
+	// since most of the go math functions require or return float64
+	beacon = Beacon{
+		// score fields
+		BeaconType:     entry.BeaconType,
+		Score:          float32(score),
+		TimestampScore: float32(tsScore),
+		DataSizeScore:  float32(dsScore),
+		HistogramScore: float32(histScore),
+		DurationScore:  float32(durScore),
+
+		// graphing fields
+		TSIntervals:      intervals,
+		TSIntervalCounts: intervalCounts,
+		DSSizes:          dsSizes,
+		DSCounts:         dsCounts,
+	}
+	return beacon, nil
+}
+
+// getBeaconScore calculates the overall beacon score from the weighted subscores
+func getBeaconScore(tsScore, tsWeight, dsScore, dsWeight, durScore, durWeight, histScore, histWeight float64) (float64, error) {
+	// ensure that the calculated subscores are between 0 and 1
+	scores := []float64{tsScore, dsScore, durScore, histScore}
+	for _, score := range scores {
+		if score < 0 || score > 1 {
+			return 0, errors.New("scores must be between 0 and 1")
+		}
+	}
+
+	// ensure that the weights are between 0 and 1 and sum to 1
+	weights := []float64{tsWeight, dsWeight, durWeight, histWeight}
+	weightSum := 0.0
+	for _, weight := range weights {
+		if weight < 0 || weight > 1 {
+			return 0, errors.New("weights must be between 0 and 1")
+		}
+		weightSum += weight
+	}
+	if weightSum != 1 {
+		return 0, errors.New("weights must sum to 1")
+	}
+
+	// calculate the final score
+	score := math.Round(((tsScore*tsWeight)+(dsScore*dsWeight)+(durScore*durWeight)+(histScore*histWeight))*1000) / 1000
+
+	return score, nil
+}
+
+func getTimestampScore(tsList []uint32) (float64, float64, float64, []int64, []int64, int64, int64, error) {
+	// ensure that the input slice has at least 4 elements (need at least 3 intervals, which requires at least 4 timestamps)
+	if len(tsList) < 4 {
+		return 0, 0, 0, nil, nil, 0, 0, fmt.Errorf("timestamp slice must contain at least 4 elements")
+	}
+
+	// find the delta times between the full, non-unique timestamp list and sort
+	// this will be used for the user/ graph reference variables returned by createCountMap
+	// the slice size is tsLength - 1 since we are looking at the deltas between timestamps
+	deltaTimesFull := make([]float64, len(tsList)-1)
+	nonZeroCounter := 0
+	for i := 0; i < len(tsList)-1; i++ {
+		interval := tsList[i+1] - tsList[i]
+		if interval > 0 {
+			nonZeroCounter++
+		}
+		deltaTimesFull[i] = float64(interval)
+	}
+
+	// ensure that there are at least 3 non-zero intervals
+	if nonZeroCounter < 3 {
+		return 0, 0, 0, nil, nil, 0, 0, fmt.Errorf("timestamp slice must contain at least 3 non-zero intervals")
+	}
+
+	// sort the delta times
+	slices.Sort(deltaTimesFull)
+
+	// get a list of the intervals found in the data, the number of times the interval was found, and the most occurring interval
+	intervals, intervalCounts, tsMode, tsModeCount, err := calculateDistinctCounts(deltaTimesFull)
+	if err != nil {
+		return 0, 0, 0, nil, nil, 0, 0, err
+	}
+
+	// deltas from the unique timestamp list are used for the scoring calculations. These can be
+	// calculated by taking the slice of the sorted deltaTimesFull from the first non-zero index
+	nonZeroIndex := 0
+
+	for i := 0; i < len(deltaTimesFull); i++ {
+		if deltaTimesFull[i] > 0 {
+			nonZeroIndex = i
+			break
+		}
+	}
+	deltaTimes := deltaTimesFull[nonZeroIndex:]
+
+	// calculate ts score, skew, and median absolute deviation
+	tsScore, tsSkew, tsMadm, err := calculateStatisticalScore(deltaTimes, 1)
+	if err != nil {
+		return 0, 0, 0, nil, nil, 0, 0, err
+	}
+
+	return tsScore, tsSkew, tsMadm, intervals, intervalCounts, tsMode, tsModeCount, nil
+
+}
+
+func getDataSizeScore(bytesList []float64) (float64, float64, float64, []int64, []int64, int64, int64, error) {
+	// ensure that the input slice has at least 3 elements
+	if len(bytesList) < 3 {
+		return 0, 0, 0, nil, nil, 0, 0, fmt.Errorf("bytes slice must contain at least 3 elements")
+	}
+
+	// sort the data sizes
+	slices.Sort(bytesList)
+
+	// find distinct data sizes and their counts
+	dsSizes, dsCounts, dsMode, dsModeCount, err := calculateDistinctCounts(bytesList)
+	if err != nil {
+		return 0, 0, 0, nil, nil, 0, 0, err
+	}
+
+	// calculate datasize score, skew, and median absolute deviation
+	dsScore, dsSkew, dsMadm, err := calculateStatisticalScore(bytesList, 0)
+	if err != nil {
+		return 0, 0, 0, nil, nil, 0, 0, err
+	}
+
+	return dsScore, dsSkew, dsMadm, dsSizes, dsCounts, dsMode, dsModeCount, nil
+
+}
+
+// calculateStatisticalScore calculates the statistical score, skew, and median absolute derivation for a given list of float64 values
+func calculateStatisticalScore(values []float64, defaultMadScore float64) (float64, float64, float64, error) {
+	// ensure that the input slice is not empty
+	if len(values) == 0 {
+		return 0, 0, 0, ErrInputSliceEmpty
+	}
+
+	// calculate the skewness of the values
+	skew, skewScore, err := calculateBowleySkewness(values)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	// calculate the median absolute deviation of the values
+	mad, madScore, err := calculateMedianAbsoluteDeviation(values, defaultMadScore)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	// calculate final statistical score
+	score := math.Round(((skewScore+madScore)/2.0)*1000) / 1000
+
+	return score, skew, mad, nil
+}
+
+// getHistogramScore calculates a score based on the histogram of timestamps of a host pair over a specified period of time
+func getHistogramScore(datasetMin int64, datasetMax int64, tsList []uint32, modeSensitivity float64, bimodalOutlierRemoval int, bimodalMinHoursSeen int, beaconTimeSpan int) ([]int, map[int32]int32, int, int, float64, error) {
+	// ensure that the input slice is not empty
+	if len(tsList) == 0 {
+		return nil, nil, 0, 0, 0, ErrInputSliceEmpty
+	}
+
+	// ensure that the dataset time range is valid
+	if datasetMax <= datasetMin {
+		return nil, nil, 0, 0, 0, ErrInvalidDatasetTimeRange
+	}
+
+	// get histogram bin eges (note: we currently look at a 24 hour period)
+	binEdges, err := computeHistogramBins(datasetMin, datasetMax, beaconTimeSpan)
+	if err != nil {
+		return nil, nil, 0, 0, 0, err
+	}
+
+	// use timestamps to get freqencies for each bin
+	freqList, freqCount, totalBars, longestRun, err := createHistogram(binEdges, tsList, modeSensitivity)
+	if err != nil {
+		return nil, nil, 0, 0, 0, err
+	}
+
+	// calculate first potential score: coefficient of variation
+	// coefficient of variation will help score histograms that have jitter in the number of
+	// connections but where the overall graph would still look relatively flat and consistent
+	// calculate coefficient of variation score
+	cvScore, err := calculateCoefficientOfVariationScore(freqList)
+	if err != nil {
+		return nil, nil, 0, 0, 0, err
+	}
+
+	// calculate second potential score: bimodal fit
+	// this will score well for graphs that have 2-3 flat sections in their connection histogram,
+	// or a bimodal freqCount histogram.
+	bimodalFitScore, err := calculateBimodalFitScore(freqCount, totalBars, bimodalOutlierRemoval, bimodalMinHoursSeen)
+	if err != nil {
+		return nil, nil, 0, 0, 0, err
+	}
+
+	// calculate final score
+	// the final score is the max of the coefficient of variation and bimodal fit scores
+	score := math.Max(cvScore, bimodalFitScore)
+
+	return freqList, freqCount, totalBars, longestRun, score, nil
+}
+
+// getDurationScore calculates a duration score based on the provided input parameters, provided that
+// a sufficient amount of hours (default threshold: 6 hours) are represented in the connection frequency histogram.
+// The duration score is derived from two potential subscores: dataset timespan coverage and consistency of connection hours
+func getDurationScore(datasetMin int64, datasetMax int64, histMin int64, histMax int64, totalBars int, longestConsecutiveRun int, minHoursThreshold int, idealNumberConsistentHours int) (float64, float64, float64, error) {
+
+	// ensure that the input values are valid
+	if minHoursThreshold < 1 || idealNumberConsistentHours < 1 || datasetMax <= datasetMin || histMax <= histMin {
+		return 0, 0, 0, fmt.Errorf("invalid input for getDurationScore: check parameter values")
+	}
+
+	// initialize the variables to hold the coverage, consistency, and final score
+	coverage, consistency, score := float64(0), float64(0), float64(0)
+
+	// check if there is enough data to calculate the duration score
+	if totalBars >= minHoursThreshold {
+
+		// calculate the dataset timespan coverage score
+		// this score reflects the proportion of time covered by the dataset in relation to the
+		// entire specified timeframe. It is calculated as:
+		//    [ timestamp of last connection - timestamp of first connection ] /
+		//    [ last timestamp of dataset - first timestamp of dataset ]
+		coverage = math.Ceil((float64(histMax-histMin)/float64(datasetMax-datasetMin))*1000) / 1000
+		if coverage > 1.0 {
+			coverage = 1.0
+		}
+
+		// calculate the consistency score
+		// this score measures the continuity of connection hours, considering the longest run
+		// of consecutive hours observed. Consecutive hours include wrap-around from the start
+		// to the end of the dataset. It is calculated as:
+		//    [ longest run of consecutive hours seen] / [ Ideal consecutive hours (default: 12) ]
+		consistency = math.Ceil((float64(longestConsecutiveRun)/float64(idealNumberConsistentHours))*1000) / 1000
+		if consistency > 1.0 {
+			consistency = 1.0
+		}
+
+		// take the maximum of the two scores
+		score = math.Max(coverage, consistency)
+	}
+
+	return coverage, consistency, score, nil
+}
+
+// calculateBowleySkewness calculates a measure of skewness for a distribution.
+// Perfect beacons would have symmetric delta time and size distributions
+func calculateBowleySkewness(data []float64) (float64, float64, error) {
+	// ensure that the input slice is not empty, since the minimum number of
+	// elements required to calculate skewness is 3
+	if len(data) < 3 {
+		return 0, 0, fmt.Errorf("input slice must not contain fewer than 3 elements")
+	}
+
+	// calculate the quartiles
+	quartiles, err := stats.Quartile(data)
+
+	// returns an error if array was empty or quartiles could not be calculated
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// calculate the numerator
+	num := quartiles.Q1 + quartiles.Q3 - 2*quartiles.Q2
+
+	// calculate the denominator
+	den := quartiles.Q3 - quartiles.Q1
+
+	// set the skewness to zero
+	skewness := float64(0)
+
+	// Bowley Skewness = (Q3+Q1 – 2Q2) / (Q3 – Q1)
+	// if the denominator less than 10 or the median is equal to the lower or upper quartile, the skewness is zero
+	if den >= 10 && quartiles.Q2 != quartiles.Q1 && quartiles.Q2 != quartiles.Q3 {
+		skewness = float64(num) / float64(den)
+	}
+
+	// calculate score
+	score := 1.0 - math.Abs(skewness)
+
+	// return the skewness and the score
+	return skewness, score, nil
+}
+
+// calculateMedianAbsoluteDeviation calculates the Median Absolute Deviation (MAD) about the median,
+// providing a score that measures the dispersion of a distribution. Perfectly consistent data would
+// result in a MAD score close to zero
+func calculateMedianAbsoluteDeviation(data []float64, defaultScore float64) (float64, float64, error) {
+	// ensure the the input slice is not empty
+	if len(data) == 0 {
+		return 0, 0, ErrInputSliceEmpty
+	}
+
+	// ensure that the input is sorted
+	if !sort.Float64sAreSorted(data) {
+		sort.Float64s(data)
+	}
+
+	// calculate the median of the input data
+	median, err := stats.Median(data)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	mad, err := stats.MedianAbsoluteDeviation(data)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// calculate the MAD score, which is a measure of how much the data deviates from its median.
+	// The MAD is normalized by dividing it by the median. The resulting score represents how
+	// consistent the data is. As the MAD increases, the score decreases, indicating more dispersion
+	score := defaultScore
+	if median >= 1 {
+		score = (median - mad) / median
+	}
+
+	// If the score is less than zero or NaN, return zero
+	if score < 0 || math.IsNaN(score) {
+		score = 0
+	}
+
+	// Return the MAD and the normalized MAD score
+	return mad, score, nil
+}
+
+// calculateDistinctCounts takes a sorted slice of numbers as input and returns
+// distinct numbers, their counts, mode, and maximum count
+func calculateDistinctCounts(input []float64) ([]int64, []int64, int64, int64, error) {
+	// ensure that the input slice has at least 2 elements
+	if len(input) < 2 {
+		return nil, nil, 0, 0, fmt.Errorf("input slice must have at least two elements")
+	}
+
+	// ensure that the input is sorted
+	if !sort.Float64sAreSorted(input) {
+		sort.Float64s(input)
+	}
+
+	// create a slice to store unique elements from the number list,
+	// starting with an empty slice (length 0) and a capacity based on
+	// the assumption that every element in input is distinct
+	distinctNumbers := make([]int64, 0, len(input))
+
+	// countsMap will map each distinct number to its count
+	countsMap := make(map[int64]int64)
+
+	// initialize with the first element of input
+	lastNumber := int64(input[0])
+	distinctNumbers = append(distinctNumbers, lastNumber)
+	countsMap[lastNumber]++
+
+	// iterate through input to identify unique elements and count occurrences
+	for _, currentNumber := range input[1:] {
+		current := int64(currentNumber)
+
+		// if the current number is different from the last one, add it to distinctNumbers
+		if lastNumber != current {
+			distinctNumbers = append(distinctNumbers, current)
+		}
+
+		// increment the count for the current number
+		countsMap[current]++
+		lastNumber = current
+	}
+
+	// prepare the results by calculating countsArray, mode, and maxCount
+	countsArray := make([]int64, len(distinctNumbers))
+	mode := distinctNumbers[0]  // assume the mode is the first distinct number
+	maxCount := countsMap[mode] // initialize maxCount with the count of the assumed mode
+
+	// find the mode and maximum count
+	for i, number := range distinctNumbers {
+		count := countsMap[number]
+		countsArray[i] = count
+
+		// update mode and maxCount if a higher count is found
+		if count > maxCount {
+			maxCount = count
+			mode = number
+		}
+	}
+
+	return distinctNumbers, countsArray, mode, maxCount, nil
+}
+
+// computeHistogramBins creates evenly spaced bins for the histogram based on the given timestamp range
+// and the desired number of bins
+func computeHistogramBins(startTime int64, endTime int64, numBins int) ([]float64, error) {
+	// ensure that the number of bins is positive
+	if numBins <= 0 {
+		return nil, errors.New("number of desired histogram bins must be greater than 0")
+	}
+
+	// ensure that time range is valid
+	if endTime <= startTime {
+		return nil, errors.New("invalid histogram time range")
+	}
+
+	// set number of bin eges. Since the edges include the endpoints,
+	// the number of edges will be one more than the number of desired bins
+	edgeCount := numBins + 1
+
+	// calculate the step size for evenly spaced bins between startTime and endTime
+	step := float64(endTime-startTime) / float64(numBins)
+
+	// create slice to store the bin edges
+	binEdges := make([]float64, edgeCount)
+
+	// explicitly set the first bin edge to startTime
+	binEdges[0] = float64(startTime)
+
+	// create evenly spaced bin edges between startTime and endTime
+	for i := 1; i < edgeCount-1; i++ {
+		binEdges[i] = float64(startTime) + (float64(i) * step)
+	}
+
+	// explicitly set the last edge to endTime
+	binEdges[edgeCount-1] = float64(endTime)
+
+	return binEdges, nil
+}
+
+// createHistogram calculates the distribution of timestamps across given bin edges
+// func createHistogram(binEdges []uint32, timestamps []uint32, modeSensitivity float64) ([]int, map[int32]int32, int, int, error) {
+func createHistogram(binEdges []float64, timestamps []uint32, modeSensitivity float64) ([]int, map[int32]int32, int, int, error) {
+	// validate input
+	if len(binEdges) < 2 {
+		return nil, nil, 0, 0, errors.New("bin edges must contain at least 2 elements")
+	}
+
+	if len(timestamps) == 0 {
+		return nil, nil, 0, 0, ErrInputSliceEmpty
+	}
+
+	// ensure that the bin edges are sorted
+	if !sort.Float64sAreSorted(binEdges) {
+		sort.Float64s(binEdges)
+	}
+
+	// ensure that the timestamps are sorted
+	if !util.UInt32sAreSorted(timestamps) {
+		util.SortUInt32s(timestamps)
+	}
+
+	// Initialize nextBinIndex with the second bin edge to start comparisons.
+	// This variable represents the upper limit of the current bin, used to determine
+	// if a timestamp falls within the current bin or if we need to move to the next bin.
+	currentBinIndex := 0
+	nextBinEdge := binEdges[currentBinIndex+1]
+
+	// calculate the number of connections that occurred within the time span represented by each bin
+	// this is basically a histogram of the number of connections that occurred within each bin
+	// i,e, for a timestamp list of [1, 5, 23, 25, 42, 45] and bin edges [0, 10, 20, 30, 40, 50],
+	// the histogram would be [2,0,2,0,2]
+	connectionHistogram := make([]int, len(binEdges)-1)
+
+	// loop over sorted timestamp list
+	for _, timestamp := range timestamps {
+
+		// increment if still in the current bin
+		if float64(timestamp) < nextBinEdge {
+			connectionHistogram[currentBinIndex]++
+			continue
+		}
+
+		// if the timestamp is greater than or equal to the next bin edge, move to the next bin
+		for j := currentBinIndex + 1; j < len(binEdges)-1; j++ {
+			currentBinIndex = j
+			nextBinEdge = binEdges[j+1]
+			if float64(timestamp) < binEdges[j+1] {
+				break
+			}
+
+		}
+
+		// increment count
+		// this will also capture and increment for a situation where the final timestamp is
+		// equal to the final bin
+		connectionHistogram[currentBinIndex]++
+	}
+
+	// get histogram frequency counts
+	freqCount, totalBars, longestRun, err := getFrequencyCounts(connectionHistogram, modeSensitivity)
+	if err != nil {
+		return nil, nil, 0, 0, err
+	}
+
+	return connectionHistogram, freqCount, totalBars, longestRun, nil
+
+}
+
+// getFrequencyCounts calculates the frequency counts for a connection histogram, essentially counting how many
+// bars of a certain height are present in the histogram. The function also calculates the longest consecutive run
+// of hours seen in the connection frequency histogram, including wrap around from start to end of dataset.
+
+// alt description:
+// calculateFrequencyCounts analyzes the histogram to calculate the frequency of each count,
+// the total number of non-empty bins, and the longest consecutive sequence of non-empty bins seen in the histogram,
+// including wrap around from start to end of dataset
+func getFrequencyCounts(connectionHistogram []int, modeSensitivity float64) (map[int32]int32, int, int, error) {
+	// ensure that the input is not empty
+	if len(connectionHistogram) == 0 {
+		return nil, 0, 0, ErrInputSliceEmpty
+	}
+
+	// count total non-zero histogram entries (total bars) and find the largest histogram entry
+	totalBars := 0
+	largestConnCount := 0
+	for _, entry := range connectionHistogram {
+		if entry > 0 {
+			totalBars++
+		}
+		if entry > largestConnCount {
+			largestConnCount = entry
+		}
+
+	}
+
+	// create a map to store the frequency counts for the connection histogram
+	freqCount := make(map[int32]int32)
+
+	// determine bin size for frequency histogram. This is expressed as a percentage of the
+	// largest connection count and controls how forgiving the bimodal analysis is to variation.
+	// This keeps us from putting multiple bars of very similar height into different bins and
+	// interpreting them as separate modes. For example, if the largest connection count is 1000 and
+	// the bimodal sensitivity is 0.05, the bin size will be 50. This means that any bars with a
+	// within 50 of each other will be grouped together. This is useful for handling small variations
+	// in connection counts that are not significant enough to be considered separate modes.
+	// the percentage is set in the rita yaml file (default: 0.05)
+	binSize := math.Ceil(float64(largestConnCount) * modeSensitivity)
+
+	// make variables to track the longest consecutive run of hours seen in the connection
+	// frequency histogram, including wrap around from start to end of dataset
+	longestRun := 0
+	currentRun := 0
+
+	// make frequency count map
+	for i := 0; i < len(connectionHistogram)*2; i++ {
+
+		// get the bar from the connection histogram, wrapping around if necessary
+		frequency := connectionHistogram[i%len(connectionHistogram)]
+
+		// track the longest run of consecutive bars seen in the connection frequency histogram
+		if frequency > 0 {
+			currentRun++
+
+		} else {
+
+			if currentRun > longestRun {
+				longestRun = currentRun
+			}
+			currentRun = 0
+
+		}
+
+		// limit calculation to the first loop through the connection histogram
+		if i < len(connectionHistogram) {
+
+			// if the bar is greater than zero, parse it into the map entry that matches its frequency
+			if frequency > 0 {
+
+				// figure out which bin to parse the frequency bar into
+				bin := int(math.Floor(float64(frequency)/binSize) * binSize)
+
+				// create or increment bin
+				if _, ok := freqCount[int32(bin)]; !ok {
+					freqCount[int32(bin)] = 1
+				} else {
+					freqCount[int32(bin)]++
+				}
+			}
+
+		}
+
+	}
+
+	if currentRun > longestRun {
+		longestRun = currentRun
+	}
+
+	// since we could end up with 2*freqListLen for the longest run if
+	// every hour has a connection, we will fix it up here.
+	if longestRun > len(connectionHistogram) {
+		longestRun = len(connectionHistogram)
+	}
+
+	return freqCount, totalBars, longestRun, nil
+}
+
+// calculateCoefficientOfVariationScore calculates the coefficient of variation score for a connection histogram.
+// The score is used to evaluate the level of jitter in the number of connections, providing a measure of how flat
+// or consistent the overall graph appears. A high coefficient of variation implies more jitter, resulting in a lower score.
+// The final score is normalized between 0 and 1, where 1 indicates perfect consistency, and 0 indicates high variation.
+//
+// potential alt description:
+// calculateCoefficientOfVariationScore calculates a score based on the coefficient of variation (CV) for a given frequency list.
+// The CV is a standardized measure of dispersion of a frequency distribution, defined as the ratio of the standard deviation to the mean.
+// This function returns a score inversely related to the CV, aiming to score datasets based on their uniformity or consistency.
+func calculateCoefficientOfVariationScore(freqList []int) (float64, error) {
+	// ensure that the input is valid
+
+	// ensure that the input slice is not empty
+	if len(freqList) == 0 {
+		return 0, ErrInputSliceEmpty
+	}
+
+	// calculate the total and check for negative values. This will also ensure that the
+	// mean cannot be zero, a case for which the CV is unreliable
+	total := 0
+	for _, entry := range freqList {
+		if entry < 0 {
+			return 0, errors.New("input slice must not contain negative values")
+		}
+		total += entry
+	}
+	if total <= 0 {
+		return 0, errors.New("total must be greater than zero")
+	}
+
+	// calculate mean
+	freqMean := float64(total) / float64(len(freqList))
+
+	// calculate standard deviation
+	sd := float64(0)
+	for j := 0; j < len(freqList); j++ {
+		sd += math.Pow(float64(freqList[j])-freqMean, 2)
+	}
+	sd = math.Sqrt(sd / float64(len(freqList)))
+
+	// calculate coefficient of variation
+	cv := sd / math.Abs(freqMean)
+
+	// ensures datasets with high variability are not given negative scores
+	var cvScore float64
+	if cv > 1.0 {
+		cvScore = 0.0
+	} else {
+		cvScore = math.Round((1.0-cv)*1000) / 1000
+	}
+
+	// ensure that the score does not exceed 1
+	if cvScore > 1.0 {
+		cvScore = 1.0
+	}
+
+	return cvScore, nil
+}
+
+// calculateBimodalFitScore calculates the bimodal fit score for a connection histogram.
+// This score is particularly useful for graphs that exhibit 2-3 flat sections in their connection histogram or a
+// bimodal frequency count histogram. It is designed to handle scenarios like a beacon alternating between low and high
+// connection counts per hour. The score is computed only if the number of total bars on the histogram is at least the
+// specified minimum (default: 11). The final score is normalized between 0 and 1, where 1 indicates a perfect fit for
+// bimodal patterns, and 0 indicates a poor fit.
+func calculateBimodalFitScore(freqCount map[int32]int32, totalBars int, modalOutlierRemoval int, minHoursForBimodalAnalysis int) (float64, error) {
+	// ensure that the input is valid
+	if len(freqCount) == 0 {
+		return 0, errors.New("frequency count map must not be empty")
+	}
+
+	// ensure that totalBars is greater than zero
+	if totalBars <= 0 {
+		return 0, errors.New("total bars must be greater than zero")
+	}
+
+	// override the minimum hours seen back to default (just under half a day) if it is less than 6 (a quarter of a day)
+	// this is to ensure that the bimodal fit score is not calculated for histograms with too few bars, as in that case
+	// a histogram with 1-2 bars will always be given a high bimoal fit score as it technically has 1-2 modes. This is also
+	// vetted when the config is loaded and should in theory not happen outside of tests
+	if minHoursForBimodalAnalysis < 6 {
+		minHoursForBimodalAnalysis = 11
+	}
+
+	// initialize bimodal fit to zero
+	modalFit := float64(0)
+
+	// check if the histogram has enough non-zero bars to analyze for bimodal patterns
+	if totalBars >= minHoursForBimodalAnalysis {
+
+		largest, secondLargest := int32(0), int32(0)
+
+		// get the top two frequency mode bars in the histogram
+		for _, value := range freqCount {
+			if value > largest {
+				secondLargest = largest
+				largest = value
+			} else if value > secondLargest {
+				secondLargest = value
+			}
+		}
+
+		// calculate the percentage of hour blocks that fit into the top two mode bins.
+		// a small buffer for the score is provided by throwing out a yaml-set number of
+		// potential outlier bins (default: 1)
+		adjustedTotalBars := math.Max(float64(totalBars-modalOutlierRemoval), 1) // ensure that the denominator is not zero
+		modalFit = float64(largest+secondLargest) / adjustedTotalBars
+	}
+
+	// calculate final score, ensuring that it does not exceed 1
+	modalFitScore := math.Round(float64(modalFit)*1000) / 1000
+	if modalFitScore > 1.0 {
+		modalFitScore = 1.0
+	}
+
+	return modalFitScore, nil
+}

analysis/spagooper.go

@@ -0,0 +1,734 @@
+package analysis
+
+import (
+	"activecm/rita/logger"
+	"activecm/rita/progressbar"
+	"activecm/rita/util"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/charmbracelet/bubbles/progress"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
+)
+
+type AnalysisResult struct {
+	// Unique connections
+	Hash                util.FixedString `ch:"hash"`
+	Src                 net.IP           `ch:"src"`
+	SrcNUID             uuid.UUID        `ch:"src_nuid"`
+	Dst                 net.IP           `ch:"dst"`
+	DstNUID             uuid.UUID        `ch:"dst_nuid"`
+	FQDN                string           `ch:"fqdn"`
+	BeaconType          string           `ch:"beacon_type"` // (sni, ip, dns)
+	Count               uint64           `ch:"count"`
+	ProxyCount          uint64           `ch:"proxy_count"`
+	OpenCount           uint64           `ch:"open_count"`
+	TSUnique            uint64           `ch:"ts_unique"` // number of unique timestamps
+	TSList              []uint32         `ch:"ts_list"`
+	TotalDuration       float64          `ch:"total_duration"`
+	OpenTotalDuration   float64          `ch:"open_total_duration"`
+	BytesList           []float64        `ch:"bytes"`
+	TotalBytes          int64            `ch:"total_bytes"`
+	PortProtoService    []string         `ch:"port_proto_service"`
+	FirstSeenHistorical time.Time        `ch:"first_seen_historical"`
+	LastSeen            time.Time        `ch:"last_seen"`
+	ServerIPs           []net.IP         `ch:"server_ips"` // array of unique destination IPs for SNI conns
+	ProxyIPs            []net.IP         `ch:"proxy_ips"`  // array of unique proxy (destination IPs) for SNI conns
+	MissingHostCount    uint64           `ch:"missing_host_count"`
+
+	// C2 OVER DNS Connection Info
+	DirectConns []net.IP `ch:"direct_conns"`
+	QueriedBy   []net.IP `ch:"queried_by"`
+
+	// Prevalence
+	PrevalenceTotal uint64  `ch:"prevalence_total"`
+	Prevalence      float32 `ch:"prevalence"`
+
+	// C2 over DNS
+	TLD            string `ch:"tld"`
+	SubdomainCount uint64 `ch:"subdomain_count"`
+
+	// Threat Intel
+	OnThreatIntel bool `ch:"on_threat_intel"`
+}
+
+func (analyzer *Analyzer) Spagoop(ctx context.Context) error {
+	logger := logger.GetLogger()
+
+	// record start time
+	start := time.Now()
+
+	queryGroup, ctx := errgroup.WithContext(ctx)
+
+	// create progress bars
+	bars := progressbar.New(ctx, []*progressbar.ProgressBar{
+		progressbar.NewBar("SNI Connection Analysis", 1, progress.New(progress.WithDefaultGradient())),
+		progressbar.NewBar("IP Connection Analysis ", 2, progress.New(progress.WithDefaultGradient())),
+		progressbar.NewBar("DNS Analysis           ", 3, progress.New(progress.WithDefaultGradient())),
+	}, []progressbar.Spinner{})
+
+	// if !analyzer.minTS.IsZero() && !analyzer.maxTS.IsZero() {
+	logger.Debug().Msg("Starting to get unique SNI connections")
+
+	queryGroup.Go(func() error {
+		// get the unique connections from the database
+		err := analyzer.ScoopSNIConns(ctx, bars)
+		// record end time
+		end := time.Since(start)
+		// print the time it took to finish
+		logger.Debug().Str("elapsed", fmt.Sprintf("%1.2fs", end.Seconds())).Msg("FINISHED SNI BEACON QUERY")
+		return err
+	})
+
+	logger.Debug().Msg("Starting to get unique IP connections")
+
+	queryGroup.Go(func() error {
+		// get the unique connections from the database
+		err := analyzer.ScoopIPConns(ctx, bars)
+		// record end time
+		end := time.Since(start)
+		// log the time it took to finish
+		logger.Debug().Str("elapsed", fmt.Sprintf("%1.2fs", end.Seconds())).Msg("FINISHED IP BEACON QUERY")
+		return err
+	})
+
+	// }
+
+	logger.Debug().Msg("Starting to get DNS connections")
+
+	queryGroup.Go(func() error {
+		// get the unique connections from the database
+		err := analyzer.ScoopDNS(ctx, bars)
+		// record end time
+		end := time.Since(start)
+		// print the time it took to finish
+		logger.Debug().Str("elapsed", fmt.Sprintf("%1.2fs", end.Seconds())).Msg("FINISHED EXPLODED DNS QUERY")
+		return err
+	})
+
+	queryGroup.Go(func() error {
+		_, err := bars.Run()
+		if err != nil {
+			fmt.Println("error running program:", err)
+			os.Exit(1)
+		}
+		return err
+	})
+
+	// // wait for the uconn queries and check if any exited with an error
+	// // Note: If any of the g.Go routines return an error, then the context will be cancelled
+	// // and other goroutines can exit if they listen for the context cancellation (ctx.Done())
+	if err := queryGroup.Wait(); err != nil {
+		close(analyzer.UconnChan)
+		logger.Error().Err(err).Msg("could not perform uconn spagoop")
+		return err
+	}
+
+	// close uconn channel to signal that all uconns have been sent
+	close(analyzer.UconnChan)
+
+	logger.Debug().Msg("Finished getting uconns")
+	return nil
+}
+
+func (analyzer *Analyzer) ScoopSNIConns(ctx context.Context, progress *tea.Program) error {
+	logger := logger.GetLogger()
+
+	// initialize progress bar variables
+	var totalSNI uint64
+	// get total number of unique hashes between sni and opensni
+	err := analyzer.Database.Conn.QueryRow(analyzer.Database.GetContext(), `
+		SELECT count() FROM (
+			SELECT DISTINCT hash FROM sniconn_tmp
+			UNION DISTINCT
+			SELECT DISTINCT hash FROM opensniconn_tmp
+		)
+	`).Scan(&totalSNI)
+	if err != nil {
+		return err
+	}
+
+	// use context to pass a call back for progress and profile info
+	chCtx := clickhouse.Context(analyzer.Database.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		// use minTSBeacon because all SNI conns have a matching conn entry and openconn data is not limited by the hour since the tables are truncated before each import
+		"min_ts":                      fmt.Sprintf("%d", analyzer.minTSBeacon.UTC().Unix()),
+		"unique_connection_threshold": fmt.Sprint(analyzer.Config.Scoring.Beacon.UniqueConnectionThreshold),
+		"network_size":                fmt.Sprint(analyzer.networkSize),
+		"rolling":                     strconv.FormatBool(analyzer.Database.Rolling),
+	}))
+	// panic(strconv.FormatBool(analyzer.Database.Rolling))
+	rows, err := analyzer.Database.Conn.Query(chCtx, `--sql
+	WITH unique_sni AS (
+		SELECT DISTINCT hash FROM sniconn_tmp
+	),
+	prevalence_counts AS (
+	    SELECT fqdn, count() as prevalence_total FROM (
+			SELECT DISTINCT fqdn, src FROM usni
+			WHERE src_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT fqdn, dst AS src FROM usni
+			WHERE dst_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			
+			UNION DISTINCT 
+			
+			SELECT DISTINCT host as fqdn, src FROM openhttp
+			WHERE src_local
+			UNION DISTINCT
+			SELECT DISTINCT host as fqdn, dst AS src FROM openhttp
+			WHERE dst_local
+			
+			UNION DISTINCT 
+			
+			SELECT DISTINCT server_name as fqdn, src FROM openssl
+			WHERE src_local
+			UNION DISTINCT 
+			SELECT DISTINCT server_name as fqdn, dst AS src FROM openssl
+			WHERE dst_local
+
+			UNION DISTINCT
+
+			SELECT DISTINCT fqdn, src FROM udns
+			WHERE src_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT fqdn, dst AS src FROM udns
+			WHERE dst_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+	    )
+	    GROUP BY fqdn
+	),  
+	sniconns AS (
+		-- Get SNI connections (HTTP + SSL for a source IP -> destination FQDN pair)
+		SELECT hash, src, src_nuid, fqdn, 
+			countMerge(count) AS conn_count, 
+			countMerge(proxy_count) AS proxy_count,
+			0 as open_count,
+			sumMerge(total_duration) AS total_duration,
+			0 AS open_duration,
+			uniqExactMerge(unique_ts_count) AS ts_unique,
+			arraySort(groupArrayMerge(86400)(ts_list)) AS ts_list, 
+			arraySort(groupArrayMerge(86400)(src_ip_bytes_list)) AS bytes,
+			sumMerge(total_ip_bytes) as total_bytes,
+			groupUniqArrayMerge(10)(server_ips) AS server_ips, 
+			groupUniqArrayMerge(10)(proxy_ips) AS proxy_ips, 
+			maxMerge(last_seen) AS last_seen,
+			minMerge(first_seen) as first_seen
+		FROM usni
+		RIGHT JOIN unique_sni USING hash
+		-- Limit query to the last 24 hours of data
+		WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+		GROUP BY hash, src, src_nuid, fqdn, proxy
+
+		UNION ALL 
+
+		-- Get Open HTTP connections
+		SELECT  hash, src, src_nuid, host as fqdn, 
+				0 as conn_count, -- openhttp uses open_count
+				countIf(method = 'CONNECT') as proxy_count,
+				countIf(multi_request = false) as open_count,
+				0 as total_duration,
+				sum(duration) as open_duration,
+				0 as ts_unique, -- set following to zero/empty since openhttp is not included in beaconing
+				[] as ts_list, 
+				[] as bytes,
+				sum(src_ip_bytes + dst_ip_bytes) as total_bytes,
+				groupUniqArrayIf(10)(dst, method != 'CONNECT') as server_ips, 
+				groupUniqArrayIf(10)(dst, method = 'CONNECT') as proxy_ips,
+				max(ts) AS last_seen,
+				min(ts) AS first_seen
+		FROM openhttp
+		-- Right join unique HTTP hashes to limit analysis to just the connections that updated in this import
+		GROUP BY hash, src, src_nuid, fqdn
+
+		UNION ALL
+
+		-- Get Open SSL connections
+		SELECT  hash, src, src_nuid, server_name as fqdn, 
+				0 as conn_count, -- openssl uses open_count
+				0 as proxy_count, 
+				count() as open_count,
+				0 as total_duration, -- openssl uses open_duration
+				sum(duration) as open_duration,
+				0 as ts_unique, -- set following to zero/empty since openssl is not included in beaconing
+				[] as ts_list,
+				[] as bytes,
+				sum(src_ip_bytes + dst_ip_bytes) as total_bytes,
+				groupUniqArray(10)(dst) as server_ips,
+				[] as proxy_ips,
+				max(ts) AS last_seen,
+				min(ts) AS first_seen
+		FROM openssl
+		GROUP BY hash, src, src_nuid, fqdn
+	),
+	historical AS (
+		SELECT min(first_seen) AS first_seen, fqdn 
+		FROM metadatabase.historical_first_seen
+		LEFT JOIN sniconns USING fqdn
+		GROUP BY fqdn
+	),
+	port_proto AS (
+		SELECT hash, groupUniqArray(20)(port_proto_service) AS port_proto_service FROM (
+			SELECT DISTINCT hash, concat(po.dst_port, ':', po.proto, ':', po.service) as port_proto_service
+			FROM port_info po
+			LEFT JOIN sniconns s ON s.hash = po.hash
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT hash, concat(dst_port, ':', proto, ':', service) FROM openhttp
+			UNION DISTINCT
+			SELECT DISTINCT hash, concat(dst_port, ':', proto, ':', service) FROM openssl
+		)
+		GROUP BY hash
+	),
+	-- Aggregate data between all union groups into final structure
+	totaled_sniconns AS (
+		SELECT s.hash AS hash, s.src AS src, s.src_nuid AS src_nuid, s.fqdn AS fqdn, 
+			sum(conn_count) AS count,
+			sum(open_count) AS open_count,
+			sum(proxy_count) AS proxy_count,
+			sum(total_duration + open_duration) AS total_duration,
+			sum(open_duration) AS open_total_duration,
+			max(ts_unique) AS ts_unique,
+			groupArrayArray(86400)(ts_list) AS ts_list,
+			groupArrayArray(86400)(bytes) AS bytes,
+			sum(total_bytes) AS total_bytes,
+			groupUniqArrayArray(10)(server_ips) AS server_ips,
+			groupUniqArrayArray(10)(proxy_ips) AS proxy_ips,
+			max(s.last_seen) AS last_seen,
+			min(s.first_seen) AS first_seen
+		FROM sniconns s
+		GROUP BY s.hash, s.src, s.src_nuid, s.fqdn
+	)
+	SELECT  s.hash AS hash, s.src AS src, s.src_nuid AS src_nuid, s.fqdn AS fqdn, 
+			if(t.fqdn != '', true, false) AS on_threat_intel,
+			prevalence_total, 
+			toFloat32(prevalence_total / {network_size:UInt64}) AS prevalence,
+			if({rolling:Bool}, h.first_seen, s.first_seen) AS first_seen_historical,
+			'sni' AS beacon_type,
+			count,
+			open_count,
+			proxy_count,
+			total_duration,
+			open_total_duration,
+			ts_unique,
+			ts_list,
+			bytes,
+			total_bytes,
+			server_ips,
+			proxy_ips,
+			last_seen,
+			po.port_proto_service as port_proto_service
+	FROM totaled_sniconns s
+	LEFT JOIN prevalence_counts USING fqdn
+	LEFT JOIN metadatabase.threat_intel t ON s.fqdn = t.fqdn 
+	LEFT JOIN historical h ON h.fqdn = s.fqdn
+	LEFT JOIN port_proto po ON s.hash = po.hash
+`)
+	if err != nil {
+		// return error and cancel all uconn analysis
+		return fmt.Errorf("could not retrieve unique SNI connections for analysis: %w", err)
+	}
+	logger.Debug().Msg("successfully retrieved SNI connections")
+
+	i := uint64(0)
+	// loop over the rows
+	for rows.Next() {
+		select {
+		// abort this function if the context was cancelled
+		case <-ctx.Done():
+			logger.Warn().Msg("cancelling SNI uconns query for analysis")
+			rows.Close()
+			return ctx.Err()
+		default:
+			var res AnalysisResult
+			if err := rows.ScanStruct(&res); err != nil {
+				// return error and cancel all uconn analysis
+				return fmt.Errorf("could not read unique SNI connection during analysis: %w", err)
+			}
+			// send the unique sni connections to the uconn analysis channel
+			analyzer.UconnChan <- res
+			if i%1000 == 0 {
+				progress.Send(progressbar.ProgressMsg{ID: 1, Percent: float64(i / totalSNI)})
+			}
+			i++
+		}
+	}
+	rows.Close()
+	progress.Send(progressbar.ProgressMsg{ID: 1, Percent: 1})
+	return nil
+}
+
+func (analyzer *Analyzer) ScoopIPConns(ctx context.Context, progress *tea.Program) error {
+	logger := logger.GetLogger()
+
+	totalRows := uint64(0)
+	hasSetTotal := false
+	chCtx := clickhouse.Context(analyzer.Database.GetContext(), clickhouse.WithProgress(func(p *clickhouse.Progress) {
+		// set the total rows for the progress bar
+		if !hasSetTotal {
+			totalRows = p.Rows
+			if totalRows == 0 {
+				progress.Send(progressbar.ProgressMsg{ID: 2, Percent: 1})
+			}
+			hasSetTotal = true
+		} else {
+			// update the progress bar
+			if totalRows > 0 {
+				progress.Send(progressbar.ProgressMsg{ID: 2, Percent: float64((totalRows - p.Rows) / totalRows)})
+			}
+			progress.Send(progressbar.ProgressMsg{ID: 2, Percent: 1})
+		}
+	}), clickhouse.WithParameters(clickhouse.Parameters{
+		// use minTSBeacon because all entries in conn are used in beaconing and openconn data is not limited by the hour since the tables are truncated before each import
+		"min_ts":                      fmt.Sprintf("%d", analyzer.minTSBeacon.UTC().Unix()),
+		"unique_connection_threshold": fmt.Sprint(analyzer.Config.Scoring.Beacon.UniqueConnectionThreshold),
+		"network_size":                fmt.Sprint(analyzer.networkSize),
+		"rolling":                     strconv.FormatBool(analyzer.Database.Rolling),
+	}))
+
+	query := `--sql
+		WITH unique_http AS (
+			SELECT DISTINCT hash FROM sniconn_tmp
+			WHERE conn_type = 'http'
+		),
+		prevalence_counts AS (
+			SELECT ip, count() as prevalence_total FROM (
+				SELECT DISTINCT if(src_local, dst, src) as ip, if(src_local, src, dst) as internal FROM uconn
+				WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+				
+				UNION DISTINCT 
+				
+				SELECT DISTINCT if(src_local, dst, src) as ip, if(src_local, src, dst) as internal FROM openconn
+			)
+			GROUP BY ip
+		),  
+		sniconns AS ( -- usni connections that will be beacons in this import
+			SELECT hash, uniqExactMerge(u.unique_ts_count) AS unique_count, countMerge(u.count) AS total_count
+			FROM usni u
+			LEFT SEMI JOIN sniconn_tmp t USING hash
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			GROUP BY hash
+			HAVING unique_count >= {unique_connection_threshold:UInt64} AND total_count < 86400
+
+		), uid_list AS ( -- list of unique Zeek UID's used by SNI beacons in this import
+			SELECT DISTINCT zeek_uid FROM sniconn_tmp
+			INNER JOIN sniconns USING hash
+			UNION DISTINCT
+			-- open conns don't need to be joined on the potential beacons list bc open conns aren't used in beaconing
+			SELECT DISTINCT zeek_uid from opensniconn_tmp
+		), filtered_hashes AS ( -- list of unique hashes for uconns that were not used by SNI beacons in this import
+			SELECT DISTINCT hash FROM uconn_tmp u
+			-- this is used instead of an anti join because we need to query hashes that aren't associated with any zeek_uids from SNI
+			LEFT JOIN uid_list ui ON u.zeek_uid = ui.zeek_uid
+			GROUP BY hash
+			HAVING countIf(u.zeek_uid = ui.zeek_uid) = 0
+			UNION DISTINCT 
+			SELECT DISTINCT hash FROM openconnhash_tmp o
+			LEFT JOIN uid_list oi ON o.zeek_uid = oi.zeek_uid
+			GROUP BY hash
+			HAVING countIf(o.zeek_uid = oi.zeek_uid) = 0
+		),
+		ip_conns AS (
+		-- Get IP connections
+		SELECT  hash, src, src_nuid,  dst, dst_nuid, src_local, dst_local,
+				countMerge(missing_host_header_count) AS missing_host_count, 
+				countMerge(count) as conn_count,
+				0 as open_count,     -- only used in openconn/openhttp
+				0 as proxy_count,    -- only used in sni/openhttp
+				sumMerge(total_duration) as total_duration,
+				toFloat64(0) as open_duration,  -- only used for openconn/openhttp
+				arraySort(groupArrayMerge(86400)(ts_list)) as ts_list,
+				uniqExactMerge(unique_ts_count) as ts_unique, -- gets unique timestamp count for uconns
+				arraySort(groupArrayMerge(86400)(src_ip_bytes_list)) as bytes,
+				sumMerge(total_ip_bytes) as total_bytes,
+				maxMerge(last_seen) as last_seen,
+				minMerge(first_seen) as first_seen
+		FROM uconn
+		-- Limit IP connections to just connections not used by a SNI beacon
+		RIGHT JOIN filtered_hashes USING hash
+		-- Limit query to the last 24 hours of data
+		WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+
+		UNION ALL
+
+		-- Get open connections
+		SELECT  hash, src, src_nuid, dst, dst_nuid, src_local, dst_local,
+				countIf(missing_host_header = true) AS missing_host_count, 
+				0 as conn_count, -- open connections use open_count
+				count() as open_count,
+				0 as proxy_count, 
+				toFloat64(0) as total_duration, -- open connections use open_duration
+				sum(duration) as open_duration,
+				[] as ts_list, -- set to zero/empty since we aren't using open connections for beaconing
+				0 as ts_unique,
+				[] as bytes,
+				sum(src_ip_bytes + dst_ip_bytes) as total_bytes,
+				min(ts) AS first_seen,
+				max(ts) AS last_seen
+		FROM openconn
+		RIGHT JOIN filtered_hashes USING hash -- exclude SNI connections
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+		),
+		-- Aggregate data between all union groups
+		totaled_ipconns AS (
+			SELECT  hash, src, src_nuid, dst, dst_nuid, src_local, dst_local,
+				sum(missing_host_count) as missing_host_count,
+				sum(conn_count) as count,
+				sum(open_count) as open_count,
+				sum(proxy_count) as proxy_count,
+				sum(total_duration + open_duration) as total_duration,
+				sum(open_duration) as open_total_duration,
+				groupArrayArray(86400)(ts_list) as ts_list,
+				-- since the uniqExact AggregateFunctions are defined on uconn and usni (2 separate materialized views),
+				-- the unique ts count doesn't represent the unique set between both uconn and usni, so we must take the max of these two
+				-- and as long as that value is greater than the unique_connection_threshold (checked when we loop through the results), 
+				-- we will send it to the beacon analysis workers
+				max(ts_unique) as ts_unique,
+				groupArrayArray(86400)(bytes) as bytes,
+				sum(total_bytes) as total_bytes,
+				max(last_seen) as last_seen,
+				min(first_seen) as first_seen
+				-- any(po.port_proto_service) as port_proto_service
+		FROM ip_conns
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+		),
+		-- historical and port_proto are split out here instead of just being joined on at the end in order to avoid
+		-- multiplying the results (cartesian product)
+		historical AS (
+			SELECT min(first_seen) AS first_seen, ip 
+			FROM metadatabase.historical_first_seen h
+			LEFT JOIN ip_conns i ON h.ip = multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) 
+			GROUP BY ip
+		),
+		port_proto AS (
+			SELECT hash, groupUniqArray(20)(port_proto_service) AS port_proto_service FROM (
+				SELECT DISTINCT hash, if(po.proto = 'icmp', concat(po.proto, ':', po.icmp_type, '/', po.icmp_code), concat(po.dst_port, ':', po.proto, ':', po.service)) as port_proto_service
+				FROM port_info po
+				LEFT JOIN ip_conns i ON i.hash = po.hash
+				WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+				UNION DISTINCT
+				SELECT DISTINCT hash, if(proto = 'icmp', concat(proto, ':', src_port, '/', dst_port), concat(dst_port, ':', proto, ':', service)) as port_proto_service
+				FROM openconn
+				WHERE missing_host_header = false
+			)
+			GROUP BY hash
+		)
+		SELECT  i.hash AS hash, i.src as src, i.src_nuid as src_nuid, i.dst as dst, i.dst_nuid as dst_nuid, 
+				'ip' AS beacon_type,
+				missing_host_count,
+				count,
+				open_count,
+				proxy_count,
+				total_duration,
+				open_total_duration,
+				ts_list,
+				ts_unique,
+				bytes,
+				total_bytes,
+				last_seen,
+				if(t.ip != '::', true, false) AS on_threat_intel,
+				prevalence_total, 
+				toFloat32(prevalence_total / {network_size:UInt64}) AS prevalence,
+				if({rolling:Bool}, h.first_seen, i.first_seen) AS first_seen_historical,
+				po.port_proto_service as port_proto_service
+		FROM totaled_ipconns i 
+		LEFT JOIN prevalence_counts p ON if(src_local = true, i.dst, i.src) = p.ip
+		LEFT JOIN metadatabase.threat_intel t ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = t.ip
+		LEFT JOIN port_proto po ON i.hash = po.hash
+		LEFT JOIN historical h ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = h.ip
+
+	`
+
+	rows, err := analyzer.Database.Conn.Query(chCtx, query)
+	if err != nil {
+		// return error and cancel all uconn analysis
+		return fmt.Errorf("could not retrieve unique IP connections for analysis: %w", err)
+	}
+	logger.Debug().Msg("successsfully retrieved IP connections")
+	// loop over the rows
+	for rows.Next() {
+		select {
+		// abort this function if the context was cancelled
+		case <-ctx.Done():
+			logger.Warn().Msg("cancelling IP uconns query for analysis")
+			rows.Close()
+			return ctx.Err()
+		default:
+			var res AnalysisResult
+			if err := rows.ScanStruct(&res); err != nil {
+				// return error and cancel all uconn analysis
+				return fmt.Errorf("could not read IP connection during analysis: %w", err)
+			}
+
+			// send the unique ip connection to the uconn analysis channel
+			analyzer.UconnChan <- res
+		}
+	}
+	rows.Close()
+	return nil
+}
+
+func (analyzer *Analyzer) ScoopDNS(ctx context.Context, progress *tea.Program) error {
+	logger := logger.GetLogger()
+
+	totalRows := uint64(0)
+	hasSetTotal := false
+
+	// use context to pass a call back for progress and profile info
+	chCtx := clickhouse.Context(analyzer.Database.GetContext(), clickhouse.WithProgress(func(p *clickhouse.Progress) {
+		// set the total rows for the progress bar
+		if !hasSetTotal {
+			totalRows = p.Rows
+			if totalRows == 0 {
+				progress.Send(progressbar.ProgressMsg{ID: 3, Percent: 1})
+			}
+			hasSetTotal = true
+		} else {
+			// update the progress bar
+			if totalRows > 0 {
+				progress.Send(progressbar.ProgressMsg{ID: 3, Percent: float64((totalRows - p.Rows) / totalRows)})
+			}
+			progress.Send(progressbar.ProgressMsg{ID: 3, Percent: 1})
+		}
+
+	}), clickhouse.WithParameters(clickhouse.Parameters{
+		// use minTS (not minTSBeacon) because DNS logs don't get correlated with conn logs
+		"min_ts":              fmt.Sprintf("%d", analyzer.minTS.UTC().Unix()),
+		"subdomain_threshold": fmt.Sprint(analyzer.Config.Scoring.C2SubdomainThreshold),
+		"rolling":             strconv.FormatBool(analyzer.Database.Rolling),
+		"network_size":        fmt.Sprint(analyzer.networkSize),
+	}))
+
+	rows, err := analyzer.Database.Conn.Query(chCtx, `--sql
+		-- use only the domains from this import to reduce computation cost
+		WITH unique_tld AS (
+			SELECT DISTINCT tld FROM dns_tmp
+		), 
+		prevalence_counts AS (
+			SELECT tld, count() AS prevalence_total FROM (
+				SELECT DISTINCT cutToFirstSignificantSubdomain(fqdn) as tld, src FROM usni
+				WHERE src_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+				UNION DISTINCT
+				SELECT DISTINCT cutToFirstSignificantSubdomain(fqdn) as tld, dst AS src FROM usni
+				WHERE dst_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+				
+				UNION DISTINCT 
+				
+				SELECT DISTINCT cutToFirstSignificantSubdomain(host) as tld, src FROM openhttp
+				WHERE src_local
+				UNION DISTINCT
+				SELECT DISTINCT cutToFirstSignificantSubdomain(host) as tld, dst AS src FROM openhttp
+				WHERE dst_local
+				
+				UNION DISTINCT 
+				
+				SELECT DISTINCT cutToFirstSignificantSubdomain(server_name) as tld, src FROM openssl
+				WHERE src_local
+				UNION DISTINCT 
+				SELECT DISTINCT cutToFirstSignificantSubdomain(server_name) as tld, dst AS src FROM openssl
+				WHERE dst_local
+
+				UNION DISTINCT
+
+				SELECT DISTINCT tld, src FROM udns
+				WHERE src_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+				UNION DISTINCT
+				SELECT DISTINCT tld, dst AS src FROM udns
+				WHERE dst_local AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			)
+			GROUP BY tld
+		),  
+		-- grab the last seen dates for the domains from this import
+		unique_dns AS (
+			SELECT tld, maxMerge(last_seen) as last_seen, minMerge(first_seen) as first_seen from udns
+			-- limiting the scope to just the domains in this import here 
+			-- has significant performance benefits as opposed to doing it later in the query
+			RIGHT JOIN unique_tld USING tld
+			GROUP BY tld
+		),
+		sussy_subdomains AS (
+			-- get all tlds with more than 100 subdomains
+			SELECT tld, uniqExactMerge(subdomains) as subdomain_count FROM exploded_dns
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			GROUP BY tld
+			HAVING subdomain_count >= 100
+		-- get all the resolved ips for the tld
+		), resolved_ips AS (
+			SELECT DISTINCT resolved_ip, tld FROM pdns
+			RIGHT JOIN sussy_subdomains USING tld
+			WHERE day >= toStartOfDay(fromUnixTimestamp({min_ts:Int64}))
+		-- get all source ips that made a connection to the resolved ips
+		), direct_connections AS (
+			SELECT tld, groupUniqArray(src) as direct_conns FROM uconn u
+			RIGHT JOIN resolved_ips r ON u.dst = r.resolved_ip
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			GROUP BY tld
+		-- get all systems that performed a dns query to the tld
+		), queried_by AS (
+			SELECT tld, groupUniqArray(src) as queried FROM udns
+			INNER JOIN sussy_subdomains USING tld
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			GROUP BY tld
+		-- keep tlds which had zero non-dns-server ips in direct connections
+		),
+		historical AS (
+			SELECT min(first_seen) AS first_seen, cutToFirstSignificantSubdomain(fqdn) as tld 
+			FROM metadatabase.historical_first_seen
+			LEFT JOIN exploded_dns USING tld
+			GROUP BY tld
+		),
+		totaled_exploded AS (
+			SELECT tld, uniqExactMerge(subdomains) AS subdomain_count
+			FROM exploded_dns
+			WHERE hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			GROUP BY tld
+			HAVING subdomain_count >= {subdomain_threshold:Int32}
+		)
+		-- get the subdomain counts and the last seen count for each tld
+		SELECT e.tld AS tld, e.subdomain_count as subdomain_count, 
+			'dns' AS beacon_type,
+			d.direct_conns as direct_conns, q.queried as queried_by, u.last_seen as last_seen,
+			prevalence_total, 
+			toFloat32(prevalence_total / {network_size:UInt64}) AS prevalence,
+			-- use the historical first seen value if this dataset is rolling
+			if({rolling:Bool}, h.first_seen, u.first_seen) AS first_seen_historical,
+			if(cutToFirstSignificantSubdomain(t.fqdn) != '', true, false) AS on_threat_intel
+		FROM totaled_exploded e
+		INNER JOIN unique_dns u ON e.tld = u.tld
+		LEFT JOIN prevalence_counts p ON e.tld = p.tld
+		LEFT JOIN historical h ON e.tld = h.tld
+		LEFT JOIN direct_connections d ON e.tld = d.tld
+		LEFT JOIN queried_by q ON e.tld = q.tld
+		LEFT JOIN metadatabase.threat_intel t ON e.tld = cutToFirstSignificantSubdomain(t.fqdn)	
+	`)
+	if err != nil {
+		// return error and cancel all uconn analysis
+		return fmt.Errorf("could not retrieve unique exploded domains for analysis: %w", err)
+	}
+	logger.Debug().Msg("successfully retrieved exploded dns")
+	// loop over the rows
+	for rows.Next() {
+		select {
+		// abort this function if the context was cancelled
+		case <-ctx.Done():
+			logger.Warn().Msg("cancelling exploded dns query for analysis")
+			rows.Close()
+			return ctx.Err()
+		default:
+			var res AnalysisResult
+			if err := rows.ScanStruct(&res); err != nil {
+				// return error and cancel all uconn analysis
+				return fmt.Errorf("could not read exploded dns during analysis: %w", err)
+			}
+			// send the unique ip connection to the uconn analysis channel
+			analyzer.UconnChan <- res
+		}
+	}
+	rows.Close()
+	return nil
+}

analysis/uconns.sql

@@ -0,0 +1,165 @@
+@set min_ts=0
+@set rolling=false
+@set network_size=10
+@set unique_thresh=4
+
+WITH unique_http AS (
+			SELECT DISTINCT hash FROM sniconn_tmp
+			WHERE conn_type = 'http'
+		),
+		prevalence_counts AS (
+			SELECT ip, count() as prevalence_total FROM (
+				SELECT DISTINCT if(src_local, dst, src) as ip, if(src_local, src, dst) as internal FROM uconn
+				WHERE hour >= toStartOfHour(fromUnixTimestamp(:min_ts))
+				
+				UNION DISTINCT 
+				
+				SELECT DISTINCT if(src_local, dst, src) as ip, if(src_local, src, dst) as internal FROM openconn
+			)
+			GROUP BY ip
+		),  
+		sniconns AS ( -- usni connections that will be beacons in this import
+			SELECT hash, uniqExactMerge(u.unique_ts_count) AS unique_count, countMerge(u.count) AS total_count
+			FROM usni u
+			LEFT SEMI JOIN sniconn_tmp t USING hash
+			WHERE hour >= toStartOfHour(fromUnixTimestamp(:min_ts))
+			GROUP BY hash
+			HAVING unique_count >= :unique_thresh AND total_count < 86400
+
+		), uid_list AS ( -- list of unique Zeek UID's used by SNI beacons in this import
+			SELECT DISTINCT zeek_uid FROM sniconn_tmp
+			INNER JOIN sniconns USING hash
+			UNION DISTINCT
+			-- open conns don't need to be joined on the potential beacons list bc open conns aren't used in beaconing
+			SELECT DISTINCT zeek_uid from opensniconn_tmp
+		), filtered_hashes AS ( -- list of unique hashes for uconns that were not used by SNI beacons in this import
+			SELECT DISTINCT hash FROM uconn_tmp u
+			-- this is used instead of an anti join because we need to query hashes that aren't associated with any zeek_uids from SNI
+			LEFT JOIN uid_list ui ON u.zeek_uid = ui.zeek_uid
+			GROUP BY hash
+			HAVING countIf(u.zeek_uid = ui.zeek_uid) = 0
+			UNION DISTINCT 
+			SELECT DISTINCT hash FROM openconnhash_tmp o
+			LEFT JOIN uid_list oi ON o.zeek_uid = oi.zeek_uid
+			GROUP BY hash
+			HAVING countIf(o.zeek_uid = oi.zeek_uid) = 0
+		),
+		ip_conns AS (
+		-- Get IP connections
+		SELECT  hash, src, src_nuid,  dst, dst_nuid, src_local, dst_local,
+				countMerge(missing_host_header_count) AS missing_host_count, 
+				countMerge(count) as conn_count,
+				0 as open_count,     -- only used in openconn/openhttp
+				0 as proxy_count,    -- only used in sni/openhttp
+				sumMerge(total_duration) as total_duration,
+				toFloat64(0) as open_duration,  -- only used for openconn/openhttp
+				arraySort(groupArrayMerge(86400)(ts_list)) as ts_list,
+				uniqExactMerge(unique_ts_count) as ts_unique, -- gets unique timestamp count for uconns
+				arraySort(groupArrayMerge(86400)(src_ip_bytes_list)) as bytes,
+				sumMerge(total_ip_bytes) as total_bytes,
+				maxMerge(last_seen) as last_seen,
+				minMerge(first_seen) as first_seen
+		FROM uconn
+		-- Limit IP connections to just connections not used by a SNI beacon
+		RIGHT JOIN filtered_hashes USING hash
+		-- Limit query to the last 24 hours of data
+		WHERE hour >= toStartOfHour(fromUnixTimestamp(:min_ts))
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+
+		UNION ALL
+
+		-- Get open connections
+		SELECT  hash, src, src_nuid, dst, dst_nuid, src_local, dst_local,
+				countIf(missing_host_header = true) AS missing_host_count, 
+				0 as conn_count, -- open connections use open_count
+				count() as open_count,
+				0 as proxy_count, 
+				toFloat64(0) as total_duration, -- open connections use open_duration
+				sum(duration) as open_duration,
+				[] as ts_list, -- set to zero/empty since we aren't using open connections for beaconing
+				0 as ts_unique,
+				[] as bytes,
+				sum(src_ip_bytes + dst_ip_bytes) as total_bytes,
+				min(ts) AS first_seen,
+				max(ts) AS last_seen
+		FROM openconn
+		RIGHT JOIN filtered_hashes USING hash -- exclude SNI connections
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+		),
+		-- Aggregate data between all union groups
+		totaled_ipconns AS (
+			SELECT  hash, src, src_nuid, dst, dst_nuid, src_local, dst_local,
+				-- if(t.ip != '::', true, false) AS on_threat_intel,
+				-- prevalence_total, 
+				-- toFloat32(prevalence_total / {network_size:UInt64}) AS prevalence,
+				-- min(if({rolling:Bool}, h.first_seen, i.first_seen)) AS first_seen_historical,
+				sum(missing_host_count) as missing_host_count,
+				sum(conn_count) as count,
+				sum(open_count) as open_count,
+				sum(proxy_count) as proxy_count,
+				sum(total_duration + open_duration) as total_duration,
+				sum(open_duration) as open_total_duration,
+				groupArrayArray(86400)(ts_list) as ts_list,
+				-- since the uniqExact AggregateFunctions are defined on uconn and usni (2 separate materialized views),
+				-- the unique ts count doesn't represent the unique set between both uconn and usni, so we must take the max of these two
+				-- and as long as that value is greater than the unique_connection_threshold (checked when we loop through the results), 
+				-- we will send it to the beacon analysis workers
+				max(ts_unique) as ts_unique,
+				groupArrayArray(86400)(bytes) as bytes,
+				sum(total_bytes) as total_bytes,
+				max(last_seen) as last_seen,
+				min(first_seen) as first_seen
+				-- any(po.port_proto_service) as port_proto_service
+		FROM ip_conns
+		-- LEFT JOIN prevalence_counts p ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = p.ip
+		-- LEFT JOIN metadatabase.threat_intel t ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = t.ip
+		-- LEFT JOIN port_proto po ON i.hash = po.hash
+		-- LEFT JOIN historical h ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = h.ip
+		GROUP BY hash, src, src_nuid, dst, dst_nuid, src_local, dst_local
+		),
+		-- historical and port_proto are split out here instead of just being joined on at the end in order to avoid
+		-- multiplying the results (cartesian product)
+		historical AS (
+			SELECT min(first_seen) AS first_seen, ip 
+			FROM metadatabase.historical_first_seen h
+			LEFT JOIN ip_conns i ON h.ip = multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) 
+			GROUP BY ip
+		),
+		port_proto AS (
+			SELECT hash, groupUniqArray(20)(port_proto_service) AS port_proto_service FROM (
+				SELECT DISTINCT hash, concat(po.dst_port, ':', po.proto, ':', po.service) as port_proto_service
+				FROM port_info po
+				LEFT JOIN ip_conns i ON i.hash = po.hash
+				WHERE hour >= toStartOfHour(fromUnixTimestamp(:min_ts))
+				UNION DISTINCT
+				SELECT DISTINCT hash, concat(dst_port, ':', proto, ':', service) as port_proto_service
+				FROM openconn
+				WHERE missing_host_header = false
+			)
+			GROUP BY hash
+		)
+		SELECT  i.hash AS hash, i.src as src, i.src_nuid as src_nuid, i.dst as dst, i.dst_nuid as dst_nuid, 
+				'ip' AS beacon_type,
+				missing_host_count,
+				count,
+				open_count,
+				proxy_count,
+				total_duration,
+				open_total_duration,
+				ts_list,
+				ts_unique,
+				bytes,
+				total_bytes,
+				last_seen,
+				if(t.ip != '::', true, false) AS on_threat_intel,
+				prevalence_total, 
+				toFloat32(prevalence_total / :network_size) AS prevalence,
+				if(:rolling, h.first_seen, i.first_seen) AS first_seen_historical,
+				po.port_proto_service as port_proto_service
+		FROM totaled_ipconns i 
+		LEFT JOIN prevalence_counts p ON if(src_local = true, i.dst, i.src) = p.ip
+		-- LEFT JOIN prevalence_counts p ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = p.ip
+		LEFT JOIN metadatabase.threat_intel t ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = t.ip
+		LEFT JOIN port_proto po ON i.hash = po.hash
+		LEFT JOIN historical h ON multiIf(src_local = true, i.dst, dst_local = true, i.src, i.dst) = h.ip
+		

cmd/cmd.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"errors"
+
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ErrMissingDatabaseName = errors.New("database name is required")
+var ErrMissingConfigPath = errors.New("config path parameter is required")
+var ErrTooManyArguments = errors.New("too many arguments provided")
+
+func Commands() []*cli.Command {
+	return []*cli.Command{
+		ImportCommand,
+		ViewCommand,
+		DeleteCommand,
+		ListCommand,
+		ValidateConfigCommand,
+	}
+}
+
+func ConfigFlag(required bool) *cli.StringFlag {
+	return &cli.StringFlag{
+		Name:     "config",
+		Aliases:  []string{"c"},
+		Usage:    "Load configuration from `FILE`",
+		Value:    "./config.hjson", // default config file path
+		Required: required,
+		Action: func(_ *cli.Context, path string) error {
+			return ValidateConfigPath(afero.NewOsFs(), path)
+		},
+	}
+}

cmd/cmd_test.go

@@ -0,0 +1,148 @@
+package cmd_test
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/joho/godotenv"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"github.com/testcontainers/testcontainers-go"
+	cl "github.com/testcontainers/testcontainers-go/modules/clickhouse"
+	"github.com/urfave/cli/v2"
+)
+
+const ConfigPath = "../integration/test_config.hjson"
+
+const TestDataPath = "../test_data"
+
+type CmdTestSuite struct {
+	suite.Suite
+	cfg                  *config.Config
+	clickhouseContainer  *cl.ClickHouseContainer
+	clickhouseConnection string
+	server               *database.ServerConn
+	// testDB               *dbTester
+}
+
+func TestMain(m *testing.M) {
+	// load environment variables with panic prevention
+	if err := godotenv.Overload("../.env", "../integration/test.env"); err != nil {
+		log.Fatalf("error loading .env file: %v", err)
+	}
+	os.Exit(m.Run())
+}
+
+func TestCmdTestSuite(t *testing.T) {
+	suite.Run(t, new(CmdTestSuite))
+}
+
+// SetupSuite is run once before the first test starts
+func (c *CmdTestSuite) SetupSuite() {
+	t := c.T()
+
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// load the config file
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err, "config should load without error")
+
+	// start clickhouse container
+	c.SetupClickHouse(t)
+
+	// update the config to use the clickhouse container connection
+	cfg.DBConnection = c.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "config should update without error")
+	c.cfg = cfg
+
+	// connect to clickhouse server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	require.NoError(t, err, "connecting to server should not produce an error")
+	c.server = server
+}
+
+// TearDownSuite is run once after all tests have finished
+func (c *CmdTestSuite) TearDownSuite() {
+	if err := c.clickhouseContainer.Terminate(context.Background()); err != nil {
+		log.Fatalf("failed to terminate clickhouse container: %s", err)
+	}
+}
+
+// SetupTest is run before each test method
+// func (d *DatabaseTestSuite) SetupTest() {}
+
+// TearDownTest is run after each test method
+// func (d *DatabaseTestSuite) TearDownTest() {}
+
+// SetupSubTest is run before each subtest
+func (c *CmdTestSuite) SetupSubTest() {
+	t := c.T()
+	fmt.Println("Running setup subtest...")
+
+	// drop all databases that may have been created during subtest
+	if c.server != nil && c.server.Conn != nil {
+		dbs, err := c.server.ListImportDatabases()
+		require.NoError(t, err, "listing databases should not produce an error")
+		for _, db := range dbs {
+			err := c.server.DeleteSensorDB(db.Name)
+			require.NoError(t, err, "dropping database should not produce an error")
+		}
+	}
+}
+
+// TearDownSubTest is run after each subtest
+// func (c *CmdTestSuite) TearDownSubTest() {}
+
+// SetupClickHouse creates a ClickHouse container using the test.docker-compose.yml and handles taking it down when complete
+func (c *CmdTestSuite) SetupClickHouse(t *testing.T) {
+	t.Helper()
+	version := os.Getenv("CLICKHOUSE_VERSION")
+	require.NotEmpty(t, version, "CLICKHOUSE_VERSION environment variable must be set")
+
+	// create ClickHouse container
+	ctx := context.Background()
+	clickHouseContainer, err := cl.RunContainer(ctx,
+		testcontainers.WithImage(fmt.Sprintf("clickhouse/clickhouse-server:%s-alpine", version)),
+		cl.WithUsername("default"),
+		cl.WithPassword(""),
+		cl.WithDatabase("default"),
+		cl.WithConfigFile(filepath.Join("../deployment/", "config.xml")),
+	)
+	require.NoError(t, err, "failed to start clickHouse container")
+
+	// get connection host
+	connectionHost, err := clickHouseContainer.ConnectionHost(ctx)
+	require.NoError(t, err, "failed to get clickHouse connection host")
+
+	// set container and connection host
+	c.clickhouseContainer = clickHouseContainer
+	c.clickhouseConnection = connectionHost
+
+}
+
+func setupTestApp(commands []*cli.Command, flags []cli.Flag) (*cli.App, context.Context) {
+	ctx := context.Background()
+
+	app := cli.NewApp()
+	app.Args = true
+	app.Commands = commands
+	app.Flags = flags
+
+	// custom exit handler to override the default which calls os.Exit
+	// this prevents the test from exiting when testing for errors
+	app.ExitErrHandler = func(_ *cli.Context, _ error) {
+		// add any custom test logic, or assertions or leave it blank
+
+	}
+
+	return app, ctx
+}

cmd/delete.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/manifoldco/promptui"
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ErrTrimmedNameEmpty = errors.New("trimmed name cannot contain wildcards or be empty")
+
+var DeleteCommand = &cli.Command{
+	Name:        "delete",
+	Usage:       "delete a dataset",
+	UsageText:   "delete [NAME]",
+	Description: "if <dataset name> ends in a wildcard, all datasets with that prefix will be deleted",
+	Args:        false,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:     "non-interactive",
+			Aliases:  []string{"ni"},
+			Usage:    "does not prompt for confirmation of deletion",
+			Value:    false,
+			Required: false,
+		},
+		ConfigFlag(false),
+	},
+	Action: func(cCtx *cli.Context) error {
+		// check if too many arguments were provided
+		if cCtx.NArg() > 1 {
+			return ErrTooManyArguments
+		}
+
+		// check if a database name was provided
+		if !cCtx.Args().Present() {
+			return ErrMissingDatabaseName
+		}
+
+		input := cCtx.Args().First()
+
+		// trim leading and trailing wildcards
+		trimmedName, err := TrimWildcards(input)
+		if err != nil {
+			return err
+		}
+
+		// set up file system interface
+		afs := afero.NewOsFs()
+
+		// validate the trimmed name
+		if err := ValidateDatabaseName(trimmedName); err != nil {
+			return err
+		}
+
+		prompt := true
+		if cCtx.Bool("non-interactive") {
+			prompt = false
+		}
+		// run the delete command
+		if err := runDeleteCmd(afs, cCtx.String("config"), input, trimmedName, prompt); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+func runDeleteCmd(afs afero.Fs, configPath string, entry string, trimmedName string, ask bool) error {
+	// validate the trimmed name
+	if len(trimmedName) == 0 {
+		return ErrTrimmedNameEmpty
+	}
+
+	// load config file
+	cfg, err := config.LoadConfig(afs, configPath)
+	if err != nil {
+		return err
+	}
+
+	// connect to server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	if err != nil {
+		return err
+	}
+
+	prompt := promptui.Prompt{
+		Label:     "Delete Dataset",
+		IsConfirm: true,
+	}
+
+	wildcardStart := strings.HasPrefix(entry, "*")
+	wildcardEnd := strings.HasSuffix(entry, "*")
+	if wildcardStart || wildcardEnd {
+		fmt.Printf("Deleting databases matching: %s\n", entry)
+		switch {
+		case wildcardStart && !wildcardEnd:
+			fmt.Printf("Deleting databases ending with: %s\n", trimmedName)
+		case !wildcardStart && wildcardEnd:
+			fmt.Printf("Deleting databases beginning with: %s\n", trimmedName)
+		case wildcardStart && wildcardEnd:
+			fmt.Printf("Deleting databases containing: %s\n", trimmedName)
+		default:
+			return errors.New("unable to determine wildcard status for dataset deletion")
+		}
+
+		if ask {
+			if _, err := prompt.Run(); err != nil {
+				fmt.Println("Cancelling deletion...")
+				return err
+			}
+		}
+
+		numDeleted, err := server.DropMultipleSensorDatabases(trimmedName, wildcardStart, wildcardEnd)
+		if err != nil {
+			return err
+		}
+		if numDeleted == 0 {
+			fmt.Println("Found no matching datasets to delete.")
+		} else {
+			fmt.Println("Successfully deleted", numDeleted, "datasets")
+		}
+	} else {
+		fmt.Printf("Deleting database: %s\n", entry)
+
+		if ask {
+			if _, err := prompt.Run(); err != nil {
+				fmt.Println("Cancelling deletion...")
+				return err
+			}
+		}
+
+		if err := server.DeleteSensorDB(entry); err != nil {
+			return err
+		}
+
+		fmt.Println("Successfully deleted dataset if it existed.")
+
+	}
+
+	return nil
+}
+
+// TrimWildcards removes leading and trailing wildcards from a database name
+func TrimWildcards(dbName string) (string, error) {
+	// regex to remove leading and trailing wildcards
+	re := regexp.MustCompile(`^\*+|\*+$`)
+	trimmedName := re.ReplaceAllString(dbName, "")
+
+	// check if the trimmed name contains any wildcards or is empty
+	if strings.Contains(trimmedName, "*") || len(trimmedName) == 0 {
+		return "", ErrTrimmedNameEmpty
+	}
+
+	return trimmedName, nil
+}

cmd/delete_test.go

@@ -0,0 +1,152 @@
+package cmd_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/database"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+)
+
+func (c *CmdTestSuite) TestDeleteCommand() {
+	commands := []*cli.Command{cmd.DeleteCommand}
+	flags := []cli.Flag{}
+
+	tests := []struct {
+		name                 string
+		args                 []string
+		dbs                  []string
+		expectedDeletedDbs   []string
+		expectedRemainingDbs []string
+		expectedError        error
+	}{
+		{
+			name:                 "No Wildcards - Database Matching Trimmed Name Exactly",
+			args:                 []string{"app", "delete", "--ni", "bingbong"},
+			dbs:                  []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedDeletedDbs:   []string{"bingbong"},
+			expectedRemainingDbs: []string{"prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedError:        nil,
+		},
+		{
+			name:                 "Prefix Wildcard - Databases Ending with Trimmed Name",
+			args:                 []string{"app", "delete", "--ni", "*bingbong"},
+			dbs:                  []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedDeletedDbs:   []string{"bingbong", "prefix_bingbong"},
+			expectedRemainingDbs: []string{"bingbong123", "prefix_bingbong123"},
+			expectedError:        nil,
+		},
+		{
+			name:                 "Suffix Wildcard - Databases Starting with Trimmed Name",
+			args:                 []string{"app", "delete", "--ni", "bingbong*"},
+			dbs:                  []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedDeletedDbs:   []string{"bingbong", "bingbong123"},
+			expectedRemainingDbs: []string{"prefix_bingbong", "prefix_bingbong123"},
+			expectedError:        nil,
+		},
+		{
+			name:                 "Both Wildcards - Databases Containing Trimmed Name",
+			args:                 []string{"app", "delete", "--ni", "*bingbong*"},
+			dbs:                  []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedDeletedDbs:   []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"},
+			expectedRemainingDbs: []string{},
+			expectedError:        nil,
+		},
+		{
+			name:          "Too Many Arguments",
+			args:          []string{"app", "delete", "dbname", "extra"},
+			expectedError: cmd.ErrTooManyArguments,
+		},
+		{
+			name:          "No Arguments",
+			args:          []string{"app", "delete"},
+			expectedError: cmd.ErrMissingDatabaseName,
+		},
+	}
+
+	for _, test := range tests {
+		c.Run(test.name, func() {
+			require := require.New(c.T())
+
+			// create a new app and context
+			app, ctx := setupTestApp(commands, flags)
+
+			// import new databases
+			for _, db := range test.dbs {
+				_, err := cmd.RunImportCmd(time.Now(), c.cfg, afero.NewOsFs(), "../test_data/open_conns/open", db, false, false)
+				require.NoError(err, "importing data should not produce an error")
+			}
+
+			// run app with test.args
+			err := app.RunContext(ctx, test.args)
+			if test.expectedError != nil {
+				require.Error(err, "error should not be nil")
+				require.Contains(err.Error(), test.expectedError.Error(), "error should contain expected value")
+			} else {
+				require.NoError(err, "error should be nil")
+			}
+
+			// validate that the expected databases were deleted
+			dbs, err := c.server.ListImportDatabases()
+			dbString := database.GetFlatDatabaseList(dbs)
+
+			require.NoError(err, "listing databases should not produce an error")
+
+			for _, db := range test.expectedDeletedDbs {
+				require.NotContains(dbString, db, "database %s should have been deleted", db)
+			}
+
+			// validate that the expected databases remain
+			for _, db := range test.expectedRemainingDbs {
+				require.Contains(dbString, db, "database %s should not have been deleted", db)
+			}
+
+		})
+	}
+
+}
+
+func TestTrimWildcards(t *testing.T) {
+	tests := []struct {
+		name          string
+		dbName        string
+		want          string
+		expectedError error
+	}{
+		{"Start Wildcard", "*bingbong", "bingbong", nil},
+		{"End Wildcard", "bingbong*", "bingbong", nil},
+		{"Both Wildcards", "*bingbong*", "bingbong", nil},
+		{"No Wildcard", "bingbong", "bingbong", nil},
+		{"Only Wildcard", "*", "", cmd.ErrTrimmedNameEmpty},
+		{"Only Wildcards", "**", "", cmd.ErrTrimmedNameEmpty},
+		{"Empty String", "", "", cmd.ErrTrimmedNameEmpty},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			trimmedName, err := cmd.TrimWildcards(test.dbName)
+			require.Equal(t, test.expectedError, err, "error should match expected value")
+			require.Equal(t, test.want, trimmedName, "trimmed name should match expected value")
+		})
+	}
+}
+
+func validateCommandsExist(t *testing.T, commands []*cli.Command, expected []string) {
+	expectedCmds := make(map[string]bool)
+	for _, expectedCmd := range expected {
+		expectedCmds[expectedCmd] = false
+	}
+	for _, cmd := range commands {
+		if _, ok := expectedCmds[cmd.Name]; ok {
+			expectedCmds[cmd.Name] = true
+		}
+	}
+	for expectedSubCmd, present := range expectedCmds {
+		if !present {
+			t.Errorf("expected (sub)command %s is missing", expectedSubCmd)
+		}
+	}
+}

cmd/import.go

@@ -0,0 +1,595 @@
+package cmd
+
+import (
+	"activecm/rita/analysis"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer"
+	"activecm/rita/logger"
+	"activecm/rita/modifier"
+	"activecm/rita/util"
+	"errors"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	numParsers   = 8 // largest impact
+	numDigesters = 8
+	numWriters   = 12 // 2nd largest impact
+)
+
+// util.Max(1, runtime.NumCPU()/2)
+var ErrInsufficientReadPermissions = errors.New("file does not have readable permission or does not exist")
+var ErrNoValidFilesFound = errors.New("no valid log files found")
+var ErrInvalidLogHourFormat = errors.New("could not parse hour from log file name - invalid format")
+var ErrInvalidLogHourRange = errors.New("could not parse hour from log file name - hour out of range")
+var ErrInvalidLogType = errors.New("incompatible log type")
+var ErrIncompatibleFileExtension = errors.New("incompatible file extension")
+var ErrSkippedDuplicateLog = errors.New("encountered file with same name but different extension, skipping file due to older last modified time")
+
+type WalkError struct {
+	Path  string
+	Error error
+}
+type HourlyZeekLogs []map[string][]string
+
+var ImportCommand = &cli.Command{
+	Name:      "import",
+	Usage:     "import zeek logs into a target database",
+	UsageText: "rita import [--database NAME] [-logs DIRECTORY] [--rolling] [--rebuild]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "database",
+			Aliases:  []string{"d"},
+			Usage:    "target database; database name should start with a lowercase letter, should contain only alphanumeric and underscores, and not end with an underscore",
+			Required: true,
+			Action: func(_ *cli.Context, name string) error {
+				return ValidateDatabaseName(name)
+			},
+		},
+		&cli.StringFlag{
+			Name:     "logs",
+			Aliases:  []string{"l"},
+			Usage:    "path to log directory",
+			Required: false,
+			Action: func(_ *cli.Context, path string) error {
+				return ValidateLogDirectory(afero.NewOsFs(), path)
+			},
+		},
+		&cli.BoolFlag{
+			Name:     "rolling",
+			Aliases:  []string{"r"},
+			Usage:    "indicates rolling import, which builds on and removes data to maintain a fixed length of time",
+			Value:    false,
+			Required: false,
+		},
+		&cli.BoolFlag{
+			Name:     "rebuild",
+			Aliases:  []string{"x"},
+			Usage:    "destroys existing database and imports given files",
+			Value:    false,
+			Required: false,
+		},
+		ConfigFlag(false),
+	},
+	Action: func(cCtx *cli.Context) error {
+		afs := afero.NewOsFs()
+
+		// load config file
+		cfg, err := config.LoadConfig(afs, cCtx.String("config"))
+		if err != nil {
+			return err
+		}
+		numParsers = int(math.Floor(math.Max(4, float64(runtime.NumCPU())/2)))
+		numDigesters = int(math.Floor(math.Max(4, float64(runtime.NumCPU())/2)))
+		numWriters = int(math.Floor(math.Max(4, float64(runtime.NumCPU())/2)))
+
+		// set the import start time in microseconds
+		startTime := time.Now()
+
+		// run import command
+		_, err = RunImportCmd(startTime, cfg, afs, cCtx.String("logs"), cCtx.String("database"), cCtx.Bool("rolling"), cCtx.Bool("rebuild"))
+		return err
+	},
+}
+
+type ImportTimestamps struct {
+	MinTS       time.Time
+	MaxTS       time.Time
+	MinTSBeacon time.Time
+	maxTSBeacon time.Time
+}
+type ImportResults struct {
+	importer.ResultCounts
+	ImportID         []util.FixedString
+	ImportTimestamps []ImportTimestamps
+}
+
+func RunImportCmd(startTime time.Time, cfg *config.Config, afs afero.Fs, logDir string, dbName string, rolling bool, rebuild bool) (ImportResults, error) {
+
+	var importResults ImportResults
+	logger := logger.GetLogger()
+
+	// keep track of the cumulative elapsed time
+	importStartedAt := startTime
+
+	logger.Info().Str("directory", logDir).Bool("rolling", rolling).Bool("rebuild", rebuild).Str("dataset", dbName).Str("started_at", importStartedAt.String()).Msg("Initiating new import...")
+
+	// load dataset relative to the current working directory
+	// this is done here instead of in the flag parsing so that anyone calling RunImportCmd will have the relative path
+	logDir, err := util.ParseRelativePath(logDir)
+	if err != nil {
+		return importResults, err
+	}
+
+	// create import database if it doesn't already exist and connect to it
+	db, err := database.SetUpNewImport(afs, cfg, dbName, rolling, rebuild)
+	if err != nil {
+		return importResults, err
+	}
+
+	// get list of hourly log maps of all days of log files in directory
+	logMap, walkErrors, err := WalkFiles(afs, logDir)
+	if err != nil {
+		return importResults, err
+	}
+
+	// log any errors that occurred during the walk
+	for _, walkErr := range walkErrors {
+		logger.Debug().Str("path", walkErr.Path).Err(walkErr.Error).Msg("file was left out of import due to error or incompatibility")
+	}
+
+	var elapsedTime int64
+	// var dayStartedAt time.Time
+
+	// loop through each day
+	for day, hourlyLogs := range logMap {
+		if len(logMap) > 1 {
+			logger.Info().Str("started_at", importStartedAt.String()).Msg(fmt.Sprintf("Importing day %d/%d", day+1, len(logMap)))
+		}
+
+		dayStart := time.Now()
+
+		// loop through each hour's log files
+		for hour, files := range hourlyLogs {
+
+			logger.Debug().Msg(fmt.Sprintf("------------- STARTING HOUR %v!! -------------", hour))
+			hourStart := time.Now()
+			// count the number of files in this hour
+			totalFileCount := 0
+			for zeekType := range files {
+				totalFileCount += len(files[zeekType])
+			}
+			// check that this hour contains files
+			// walkFiles errors if it found no files
+			// GetHourlyLogMap errors if it has no files left in any hour after filtering out invalid combinations of files
+			// We still need to skip importing if there are no files for this hour
+			if totalFileCount < 1 {
+				logger.Debug().Str("hour", fmt.Sprint(hour)).Msg("no valid files were selected for this hour's import")
+				// don't exit the rest of the import just because this hour doesn't contain any logs
+				continue
+			}
+
+			logger.Debug().Str("started_at", importStartedAt.String()).Msg(fmt.Sprintf("Importing hour %d/%d", hour+1, len(hourlyLogs)))
+
+			err = db.ResetTemporaryTables()
+			if err != nil {
+				return importResults, err
+			}
+
+			// parse logs
+			// importStart := importStartedAt
+			// if hour > 0 || day > 0 {
+			// 	// add the duration of all imports up to now to the original importStartedAt date
+			// 	importStart = importStartedAt.Add(time.Duration(elapsedTime) * time.Nanosecond)
+			// }
+			importer, err := importer.NewImporter(db, cfg, importStartedAt, numDigesters, numParsers, numWriters)
+			if err != nil {
+				return importResults, err
+			}
+
+			err = importer.Import(afs, files)
+			if err != nil {
+				return importResults, err
+			}
+
+			// update result counts (used for testing)
+			importResults.Conn += importer.ResultCounts.Conn
+			importResults.OpenConn += importer.ResultCounts.OpenConn
+			importResults.HTTP += importer.ResultCounts.HTTP
+			importResults.OpenHTTP += importer.ResultCounts.OpenHTTP
+			importResults.DNS += importer.ResultCounts.DNS
+			importResults.UDNS += importer.ResultCounts.UDNS
+			importResults.PDNSRaw += importer.ResultCounts.PDNSRaw
+			importResults.SSL += importer.ResultCounts.SSL
+			importResults.OpenSSL += importer.ResultCounts.OpenSSL
+			importResults.ImportID = append(importResults.ImportID, importer.ImportID)
+			logger.Debug().Msg("------------- RUNNING ANALYSIS!! -------------")
+
+			// TODO pull useCurrentTime out of beacon?
+			minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+			missingBeaconTS := errors.Is(err, database.ErrInvalidMinMaxTimestamp)
+			if err != nil && !missingBeaconTS {
+				return importResults, fmt.Errorf("could not find min/max timestamps for beaconing analysis: %w", err)
+			}
+
+			minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+			if err != nil {
+				return importResults, fmt.Errorf("could not find imported data. Be sure to include your internal subnets in 'filter.internal_subnets' in config.hjson.\n(err: %w)", err)
+			}
+
+			importResults.ImportTimestamps = append(importResults.ImportTimestamps, ImportTimestamps{
+				MinTS:       minTS,
+				MaxTS:       maxTS,
+				MinTSBeacon: minTSBeacon,
+				maxTSBeacon: maxTSBeacon,
+			})
+
+			logger.Debug().Time("min_ts", minTS).Time("max_ts", maxTS).Time("min_beacon_ts", minTSBeacon).Time("max_beacon_ts", maxTSBeacon).Bool("skip_beaconing", missingBeaconTS).Msg("timestamps used in analysis")
+
+			// set up new analyzer
+			analyzer, err := analysis.NewAnalyzer(db, cfg, importer.ImportID, minTS, maxTS, minTSBeacon, maxTSBeacon, useCurrentTime, missingBeaconTS)
+			if err != nil {
+				return importResults, err
+			}
+
+			// analyze the data
+			err = analyzer.Analyze()
+			if err != nil {
+				return importResults, err
+			}
+
+			// set up new modifier
+			modifier, err := modifier.NewModifier(db, cfg, importer.ImportID, minTS, maxTS)
+			if err != nil {
+				return importResults, err
+			}
+
+			// modify the data
+			err = modifier.Modify()
+			if err != nil {
+				return importResults, err
+			}
+
+			// add import finished record to metadatabase
+			err = db.AddImportFinishedRecordToMetaDB(importer.ImportID, minTS, maxTS)
+			if err != nil {
+				return importResults, err
+			}
+
+			// get the elapsed time for this hour
+			elapsedTime += time.Since(hourStart).Nanoseconds()
+
+			// add the duration of this hour's import to the importStartedAt time for the next import
+			importStartedAt = importStartedAt.Add(time.Duration(elapsedTime) * time.Nanosecond)
+
+			logger.Info().Str("elapsed_time", time.Since(hourStart).String()).Int("day", day).Int("hour", hour).Msg("Finished Importing Hour Chunk")
+
+		}
+
+		// only print finish message per day if there were multiple days
+		if len(logMap) > 1 {
+			logger.Info().Str("elapsed_time", time.Since(dayStart).String()).Int("day", day).Msg("Finished Importing Day")
+		}
+	}
+
+	logger.Info().Str("elapsed_time", fmt.Sprintf("%1.1fs", time.Since(startTime).Seconds())).Msg("🎊✨ Finished Import! ✨🎊")
+
+	return importResults, nil
+}
+
+func ValidateLogDirectory(afs afero.Fs, logDir string) error {
+	if logDir == "" {
+		return fmt.Errorf("log directory flag is required")
+	}
+
+	dir, err := util.ParseRelativePath(logDir)
+	if err != nil {
+		return err
+	}
+
+	// check if directory exists
+	if err := util.ValidateDirectory(afs, dir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ValidateDatabaseName(name string) error {
+	if name == "" {
+		return ErrMissingDatabaseName
+	}
+
+	// regex to validate dataset name
+	re := regexp.MustCompile("^[a-z]{1}([A-Za-z_0-9])+[A-Za-z0-9]$")
+
+	switch {
+	case len(name) > 63:
+		return fmt.Errorf("\n\t[!] database name cannot exceed 63 characters: %v", name)
+	case name == "default" || name == "system" || name == "information_schema" || name == "metadatabase":
+		return fmt.Errorf("\n\t[!] database name cannot be reserved word %v", name)
+	case unicode.IsUpper(rune(name[0])):
+		return fmt.Errorf("\n\t[!] database name must start with a lowercase letter %v", name)
+	case strings.Contains(name, "-"):
+		return fmt.Errorf("\n\t[!] database name cannot contain hyphens %v", name)
+	case !re.MatchString(name):
+		return fmt.Errorf("\n\t[!] database name is invalid. %v", name)
+
+	}
+	return nil
+}
+
+func parseFolderDate(folder string) (time.Time, error) {
+	// check if the path is a directory
+	folderDate, err := time.Parse(time.DateOnly, folder)
+	if err != nil {
+		// put non-date folders into some generic folder like 2006-01-02
+		folderDate, err = time.Parse(time.DateOnly, "2006-01-02")
+		if err != nil {
+			return time.Unix(0, 0), err
+		}
+	}
+	return folderDate, nil
+}
+
+// WalkFiles starts a goroutine to walk the directory tree at root and send the
+// path of each regular file on the string channel.  It sends the result of the
+// walk on the error channel.  If done is closed, WalkFiles abandons its work.
+func WalkFiles(afs afero.Fs, root string) ([]HourlyZeekLogs, []WalkError, error) {
+	logger := logger.GetLogger()
+
+	// check if root is a valid directory or file
+	err := util.ValidateDirectory(afs, root)
+	if err != nil && !errors.Is(err, util.ErrPathIsNotDir) {
+		return nil, nil, err
+	}
+	if err != nil && errors.Is(err, util.ErrPathIsNotDir) {
+		if err := util.ValidateFile(afs, root); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	logMap := make(map[time.Time]HourlyZeekLogs)
+
+	totalFilesFound, hour0FilesFound := 0, 0
+
+	type fileTrack struct {
+		lastModified time.Time
+		path         string
+	}
+	fTracker := make(map[string]fileTrack)
+
+	var walkErrors []WalkError
+
+	err = afero.Walk(afs, root, func(path string, info os.FileInfo, afErr error) error {
+
+		// check if afero failed to access or find a file or directory
+		if afErr != nil {
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: afErr})
+			return nil //nolint:nilerr // log the issue and continue walking
+		}
+
+		// skip if path is a directory
+		if info.IsDir() {
+			return nil
+		}
+
+		// skip if file is not a compatible log file
+		if !(strings.HasSuffix(path, ".log") || strings.HasSuffix(path, ".gz")) {
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrIncompatibleFileExtension})
+			return nil // log the issue and continue walking
+		}
+
+		// check if the file is readable
+		_, err := afs.Open(path)
+		if err != nil || !(info.Mode().Perm()&0444 == 0444) {
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrInsufficientReadPermissions})
+			return nil //nolint:nilerr // log the issue and continue walking
+		}
+
+		// trim the path name to remove the file extensions, only to leave .log
+		trimmedFileName := strings.TrimSuffix(path, ".gz")
+
+		// check if path doesn't have .log suffix anymore and add it if not
+		if !strings.HasSuffix(trimmedFileName, ".log") {
+			trimmedFileName += ".log"
+		}
+
+		// check if the file entry exists and get the existing entry if it does
+		fileData, exists := fTracker[trimmedFileName]
+
+		switch {
+		// add file if it hasn't been seen before
+		case !exists:
+			fTracker[trimmedFileName] = fileTrack{
+				lastModified: info.ModTime(),
+				path:         path,
+			}
+		// if trimmed version of the file exists in the map and the currently marked file for import
+		// was last modified more recently than this current file, replace it with this file
+		case exists && fileData.lastModified.Before(info.ModTime()):
+
+			// warn the user so that this isn't a silent operation
+			walkErrors = append(walkErrors, WalkError{Path: fTracker[trimmedFileName].path, Error: ErrSkippedDuplicateLog})
+			// logger.Warn().Str("original_path", fTracker[trimmedFileName].path).Str("replacement_path", path).Msg("encountered file with same name but different extension, potential duplicate log, skipping")
+
+			fTracker[trimmedFileName] = fileTrack{
+				lastModified: info.ModTime(),
+				path:         path,
+			}
+		// if the current file is older than the one we have already seen or no other conditions are met, skip it
+		default:
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrSkippedDuplicateLog})
+
+		}
+
+		return nil
+	})
+
+	// return an error if the file walk failed completely
+	if err != nil {
+		return nil, nil, fmt.Errorf("file walk failed: %w", err)
+	}
+
+	// group files into arrays by their log type
+	for _, file := range fTracker {
+		path := file.path
+
+		// check if the file is one of the accepted log types
+		var prefix string
+		switch {
+		case strings.HasPrefix(filepath.Base(path), importer.ConnPrefix) && !strings.HasPrefix(filepath.Base(path), importer.ConnSummaryPrefixUnderscore) && !strings.HasPrefix(filepath.Base(path), importer.ConnSummaryPrefixHyphen):
+			prefix = importer.ConnPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.OpenConnPrefix):
+			prefix = importer.OpenConnPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.DNSPrefix):
+			prefix = importer.DNSPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.HTTPPrefix):
+			prefix = importer.HTTPPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.OpenHTTPPrefix):
+			prefix = importer.OpenHTTPPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.SSLPrefix):
+			prefix = importer.SSLPrefix
+		case strings.HasPrefix(filepath.Base(path), importer.OpenSSLPrefix):
+			prefix = importer.OpenSSLPrefix
+		default: // skip file if it doesn't match any of the accepted prefixes
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: ErrInvalidLogType})
+			continue
+		}
+
+		// parse the hour from the filename
+		hour, err := ParseHourFromFilename(file.path)
+		if err != nil {
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: err})
+			continue
+		}
+
+		parentDir := filepath.Base(filepath.Dir(file.path))
+		folderDate, err := parseFolderDate(parentDir)
+		if err != nil {
+			walkErrors = append(walkErrors, WalkError{Path: path, Error: err})
+		}
+
+		// Check if the entry for the day exists, if not, initialize it
+		if _, ok := logMap[folderDate]; !ok {
+			logMap[folderDate] = make(HourlyZeekLogs, 24)
+		}
+
+		// Check if the entry for the hour exists, if not, initialize it
+		if logMap[folderDate][hour] == nil {
+			logMap[folderDate][hour] = make(map[string][]string)
+		}
+
+		// add the file to the hour map
+		logMap[folderDate][hour][prefix] = append(logMap[folderDate][hour][prefix], path)
+
+	}
+
+	// filter out invalid file combinations
+
+	// loop over each day in the log map
+	for day := range logMap {
+
+		// loop over each hour in the day
+		for hour := range logMap[day] {
+
+			// if there are no conn logs in the hour, we have to skip any SSL and HTTP logs for that hour
+			if len(logMap[day][hour][importer.ConnPrefix]) == 0 && (len(logMap[day][hour][importer.SSLPrefix]) > 0 || len(logMap[day][hour][importer.HTTPPrefix]) > 0) {
+				logger.Warn().Msg("SSL / HTTP logs are present, but no conn logs exist, skipping SSL / HTTP logs...")
+				delete(logMap[day][hour], importer.SSLPrefix)
+				delete(logMap[day][hour], importer.HTTPPrefix)
+			}
+
+			// 	// if there are no open conn logs in the hour, we have to skip any open SSL and open HTTP logs for that hour
+			if len(logMap[day][hour][importer.OpenConnPrefix]) == 0 && (len(logMap[day][hour][importer.OpenSSLPrefix]) > 0 || len(logMap[day][hour][importer.OpenHTTPPrefix]) > 0) {
+				logger.Warn().Msg("Open SSL / open HTTP logs are present, but no conn logs exist, skipping open SSL / open HTTP logs...")
+				delete(logMap[day][hour], importer.OpenSSLPrefix)
+				delete(logMap[day][hour], importer.OpenHTTPPrefix)
+			}
+
+			// track the total number of files after filtering out invalid file combinations
+			for zeekType := range logMap[day][hour] {
+				// sort the files for each log type, necessary for tests
+				slices.Sort(logMap[day][hour][zeekType])
+				totalFilesFound += len(logMap[day][hour][zeekType])
+				if hour == 0 {
+					hour0FilesFound += len(logMap[day][hour][zeekType])
+				}
+			}
+		}
+	}
+
+	// return an error if no files were found
+	if totalFilesFound == 0 {
+		return nil, walkErrors, ErrNoValidFilesFound
+	}
+
+	var importLogs []HourlyZeekLogs
+
+	var days []time.Time
+	for day := range logMap {
+		days = append(days, day)
+	}
+	slices.SortFunc(days, func(a, b time.Time) int { return a.Compare(b) })
+
+	for _, day := range days {
+		importLogs = append(importLogs, logMap[day])
+	}
+
+	return importLogs, walkErrors, err
+}
+
+// ParseHourFromFilename extracts the hour from a given filename
+func ParseHourFromFilename(filename string) (int, error) {
+	// define regex patterns to extract the hour from the filename
+	timePattern := `[A-Za-z]+\.(\d{2})[:/_]\d{2}`
+
+	// compile the timeRegex
+	timeRegex := regexp.MustCompile(timePattern)
+
+	// attempt to find a match in the filename
+	matches := timeRegex.FindStringSubmatch(filename)
+
+	// if hour pattern didn't match, check if the filename is a simple log file
+	if matches == nil {
+		// regex to identify simple log files (ie, conn.log, open_conn.log, /logs/conn.log.gz, etc) without hour
+		simpleLogPattern := `^\w+\.log(\.gz)?$`
+		simpleLogRegex := regexp.MustCompile(simpleLogPattern)
+
+		// if the filename matches the simple log pattern, consider file as 0 hour and return
+		if simpleLogRegex.MatchString(filepath.Base(filename)) {
+			return 0, nil
+		}
+
+		// if format doesn't match the hourly pattern or the simple log pattern, return an error
+		// to catch malformed hour formats
+		return 0, ErrInvalidLogHourFormat
+	}
+
+	// convert the extracted hour string to an integer
+	hour, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0, ErrInvalidLogHourFormat
+	}
+
+	// ensure the hour is in the 0-23 range
+	if hour < 0 || hour > 23 {
+		return 0, ErrInvalidLogHourRange
+	}
+
+	return hour, nil
+}

cmd/import_test.go

@@ -0,0 +1,973 @@
+package cmd_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/database"
+	"activecm/rita/util"
+	"context"
+	"fmt"
+
+	"activecm/rita/importer"
+	iofs "io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+/*
+REAL LIFE ZEEK
+
+- /opt/zeek/logs
+  - sensor1
+  - 2024-05-01
+  - 2024-05-02
+
+- /opt/zeek/logs
+  - 2024-05-01
+  - 2024-05-02
+
+NON-ROLLING LOGS
+- some folder
+  - conn.log
+  - dns.log
+  - http.log
+  - ssl.log
+
+- open_conn.log
+
+- some folder
+  - conn.log
+  - sensor1
+  - conn.log
+  - 2024-05-01
+  - 2024-05-02
+*/
+
+func (c *CmdTestSuite) TestRunImportCmd() {
+
+	type TestCase struct {
+		name           string
+		afs            afero.Fs
+		dbName         string
+		rolling        bool
+		rebuild        bool
+		logDir         string
+		hours          [][]string
+		expectedImport int
+	}
+
+	testCases := []TestCase{
+		{
+			name:           "No Subdirectories, No Hours",
+			afs:            afero.NewOsFs(),
+			dbName:         "bingbong",
+			rolling:        false,
+			rebuild:        false,
+			logDir:         "../test_data/valid_tsv",
+			hours:          [][]string{{"conn.log.gz", "dns.log.gz", "http.log.gz", "ssl.log.gz", "open_conn.log.gz", "open_http.log.gz", "open_ssl.log.gz"}},
+			expectedImport: 1,
+		},
+		{
+			name:    "Simple, SubDirectories - Multi-Day Logs",
+			afs:     afero.NewMemMapFs(),
+			dbName:  "bingbong",
+			rolling: false,
+			rebuild: false,
+			logDir:  "/logs",
+			hours: [][]string{
+				{"2024-04-29/conn.log", "2024-04-29/dns.log", "2024-04-29/http.log", "2024-04-29/ssl.log", "2024-04-29/open_conn.log", "2024-04-29/open_http.log", "2024-04-29/open_ssl.log"},
+				{"2024-05-01/conn.log", "2024-05-01/dns.log", "2024-05-01/http.log", "2024-05-01/ssl.log", "2024-05-01/open_conn.log", "2024-05-01/open_http.log", "2024-05-01/open_ssl.log", "2024-05-01/ssl_blue.log"},
+			},
+			expectedImport: 2,
+		},
+		{
+			name:    "SubDirectories, Multi-Day, Multi-Hour Logs",
+			afs:     afero.NewMemMapFs(),
+			dbName:  "bingbong",
+			rolling: false,
+			rebuild: false,
+			logDir:  "/logs",
+			hours: [][]string{
+				{"2024-04-29/conn.00:00:00-01:00:00.log", "2024-04-29/open_conn.00:00:00-01:00:00.log", "2024-04-29/dns.00:00:00-01:00:00.log", "2024-04-29/http.00:00:00-01:00:00.log", "2024-04-29/open_http.00:00:00-01:00:00.log", "2024-04-29/ssl.00:00:00-01:00:00.log", "2024-04-29/open_ssl.00:00:00-01:00:00.log"},
+				{"2024-04-29/conn.23:00:00-00:00:00.log", "2024-04-29/open_conn.23:00:00-00:00:00.log", "2024-04-29/dns.23:00:00-00:00:00.log", "2024-04-29/http.23:00:00-00:00:00.log", "2024-04-29/open_http.23:00:00-00:00:00.log", "2024-04-29/ssl.23:00:00-00:00:00.log", "2024-04-29/open_ssl.23:00:00-00:00:00.log"},
+				{"2024-05-01/conn.00:00:00-01:00:00.log", "2024-05-01/open_conn.00:00:00-01:00:00.log", "2024-05-01/dns.00:00:00-01:00:00.log", "2024-05-01/http.00:00:00-01:00:00.log", "2024-05-01/open_http.00:00:00-01:00:00.log", "2024-05-01/ssl.00:00:00-01:00:00.log", "2024-05-01/open_ssl.00:00:00-01:00:00.log"},
+				{"2024-05-01/conn.23:00:00-00:00:00.log", "2024-05-01/open_conn.23:00:00-00:00:00.log", "2024-05-01/dns.23:00:00-00:00:00.log", "2024-05-01/http.23:00:00-00:00:00.log", "2024-05-01/open_http.23:00:00-00:00:00.log", "2024-05-01/ssl.23:00:00-00:00:00.log", "2024-05-01/open_ssl.23:00:00-00:00:00.log", "2024-05-01/ssl_blue.23:00:00-00:00:00.log"},
+			},
+			expectedImport: 4,
+		},
+	}
+
+	for _, tc := range testCases {
+		c.Run(tc.name, func() {
+			t := c.T()
+
+			importStartedAt := time.Now()
+
+			var files []string
+			var fullPathHours [][]string
+
+			// get the root directory path
+			fullRootDir := tc.logDir
+
+			// if we are using the real logs directory, we need to get the real full path
+			if tc.logDir != "/logs" {
+				// get the current working directory
+				cwd, err := os.Getwd()
+				if err != nil {
+					fmt.Println("Error getting current working directory:", err)
+					return
+				}
+
+				fullRootDir = filepath.Join(cwd, tc.logDir)
+			}
+
+			// iterate over each day of logs
+			for _, day := range tc.hours {
+				// append all the files to a single list for creation
+				files = append(files, day...)
+
+				// convert the day list of files to full path versions
+				var fullHourFiles []string
+				for _, file := range day {
+					fullPath := filepath.Join(fullRootDir, file)
+					fullHourFiles = append(fullHourFiles, fullPath)
+				}
+				fullPathHours = append(fullPathHours, fullHourFiles)
+			}
+
+			// if we are using the mock directory, we need to create it along with the files
+			if tc.logDir == "/logs" {
+				// create mock directory with files
+				createMockZeekLogs(t, tc.afs, tc.logDir, files, true)
+			}
+
+			// run the import command
+			importResults, err := cmd.RunImportCmd(importStartedAt, c.cfg, tc.afs, tc.logDir, tc.dbName, tc.rolling, tc.rebuild)
+			require.NoError(t, err, "running import command should not produce an error")
+			require.NotNil(t, importResults, "import results should not be nil")
+
+			// verify the number of import IDs
+			require.Len(t, importResults.ImportID, tc.expectedImport, "import results should have expected number of import IDs")
+
+			// check if the database exists
+			exists, err := database.SensorDatabaseExists(c.server.Conn, context.Background(), tc.dbName)
+			require.NoError(t, err, "checking if sensor database exists should not produce an error")
+			require.True(t, exists, "sensor database should exist")
+
+			// check rolling status
+			isRolling, err := database.GetRollingStatus(context.Background(), c.server.Conn, tc.dbName)
+			require.NoError(t, err, "checking if sensor database is rolling should not produce an error")
+			require.Equal(t, tc.rolling, isRolling, "rolling status should match expected value")
+
+			// verify imported paths for each hour
+			for i := range fullPathHours {
+				var result struct {
+					Paths []string `ch:"paths"`
+				}
+
+				ctx := clickhouse.Context(context.Background(), clickhouse.WithParameters(clickhouse.Parameters{
+					"import_id": importResults.ImportID[i].Hex(),
+					"database":  tc.dbName,
+				}))
+
+				err = c.server.Conn.QueryRow(ctx, `
+					SELECT groupArray(path) AS paths
+					FROM metadatabase.files
+					WHERE import_id = unhex({import_id:String}) AND database = {database:String}
+				`).ScanStruct(&result)
+				require.NoError(t, err, "querying for total file count should not produce an error")
+
+				require.ElementsMatch(t, fullPathHours[i], result.Paths, "paths should match expected value")
+			}
+
+			// clean up the directory
+			if tc.logDir == "/logs" {
+				require.NoError(t, tc.afs.RemoveAll(tc.logDir), "removing directory should not produce an error")
+			}
+		})
+	}
+
+}
+
+// createMockZeekLogs creates a directory with files that contain mock Zeek logs, filling them with valid
+// log values if necessary for the test
+func createMockZeekLogs(t *testing.T, afs afero.Fs, directory string, files []string, valid bool) {
+	t.Helper()
+
+	// create directory
+	err := afs.MkdirAll(directory, os.FileMode(0o775))
+	require.NoError(t, err, "creating directory should not produce an error")
+
+	// create files
+	for _, file := range files {
+		data := []byte("test")
+		if valid {
+			data = []byte("#separator \\x09\n" +
+				"#set_separator\t,\n" +
+				"#empty_field\t(empty)\n" +
+				"#unset_field\t-\n" +
+				"#path\tconn\n" +
+				"#open\t2019-02-28-12-07-01\n" +
+				"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+				"#types\ttime\tstring\taddr\taddr\n" +
+				"1715640994.367201\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+				"1715640994.367201\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+				"1715641054.367201\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+				"1715641054.367201\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+				"1715641114.367201\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+				"1715641114.367201\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+				"1715641174.367201\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+				"1715641174.367201\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+				"1715641234.367201\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+				"1715641234.367201\tCxT125\t10.0.0.5\t52.12.0.5\n",
+			)
+		}
+		err = afero.WriteFile(afs, filepath.Join(directory, file), data, os.FileMode(0o775))
+		require.NoError(t, err, "creating files should not produce an error")
+	}
+}
+
+func createExpectedResults(logs []cmd.HourlyZeekLogs) []cmd.HourlyZeekLogs {
+	var data []cmd.HourlyZeekLogs
+
+	// create 24 hours for each day defined in test
+	for range logs {
+		hourly := make(cmd.HourlyZeekLogs, 24)
+		data = append(data, hourly)
+	}
+
+	// override the empty data structure with the test data
+	for i, day := range logs {
+		for j, hour := range day {
+			for logPrefix, logPaths := range hour {
+				// initialize map if the test data has data for that hour
+				if data[i][j] == nil {
+					data[i][j] = make(map[string][]string)
+				}
+				data[i][j][logPrefix] = logPaths
+			}
+		}
+	}
+	return data
+}
+
+func TestWalkFiles(t *testing.T) {
+	afs := afero.NewMemMapFs()
+
+	tests := []struct {
+		name                 string
+		directory            string
+		directoryPermissions iofs.FileMode
+		filePermissions      iofs.FileMode
+		subdirectories       []string
+		files                []string
+		expectedFiles        []cmd.HourlyZeekLogs
+		expectedWalkErrors   []cmd.WalkError
+		expectedError        error
+	}{
+		{
+			name:                 "Valid Non-Hour Logs",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log",
+				"conn_red.log", "dns_red.log", "http_red.log", "ssl_red.log",
+				"conn_blue.log.gz", "dns_blue.log.gz", "http_blue.log.gz", "ssl_blue.log.gz",
+				".DS_STORE", "capture_loss.16:00:00-17:00:00.log.gz", "stats.16:00:00-17:00:00.log.gz", "x509.16:00:00-17:00:00.log.gz",
+				"known_certs.16:00:00-17:00:00.log.gz",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/conn.log", "/logs/conn_blue.log.gz", "/logs/conn_red.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.log", "/logs/dns_blue.log.gz", "/logs/dns_red.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.log", "/logs/http_blue.log.gz", "/logs/http_red.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.log", "/logs/ssl_blue.log.gz", "/logs/ssl_red.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs/.DS_STORE", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/capture_loss.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/stats.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/x509.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/known_certs.16:00:00-17:00:00.log.gz", Error: cmd.ErrInvalidLogType},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                 "Hour Logs, Format 00:00:00",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				"conn.00:00:00-01:00:00.log", "conn_red.00:00:00-01:00:00.log", "conn_blue.00:00:00-01:00:00.log.gz", "open_conn.00:00:00-01:00:00.log",
+				"dns.00:00:00-01:00:00.log", "dns_red.00:00:00-01:00:00.log", "dns_blue.00:00:00-01:00:00.log.gz",
+				"http.00:00:00-01:00:00.log", "http_red.00:00:00-01:00:00.log", "http_blue.00:00:00-01:00:00.log.gz", "open_http.00:00:00-01:00:00.log",
+				"ssl.00:00:00-01:00:00.log", "ssl_red.00:00:00-01:00:00.log", "ssl_blue.00:00:00-01:00:00.log.gz", "open_ssl.00:00:00-01:00:00.log",
+
+				"conn.01:00:00-02:00:00.log", "conn_red.01:00:00-02:00:00.log", "conn_blue.01:00:00-02:00:00.log.gz", "open_conn.01:00:00-02:00:00.log",
+				"dns.01:00:00-02:00:00.log", "dns_red.01:00:00-02:00:00.log", "dns_blue.01:00:00-02:00:00.log.gz",
+				"http.01:00:00-02:00:00.log", "http_red.01:00:00-02:00:00.log", "http_blue.01:00:00-02:00:00.log.gz", "open_http.01:00:00-02:00:00.log",
+				"ssl.01:00:00-02:00:00.log", "ssl_red.01:00:00-02:00:00.log", "ssl_blue.01:00:00-02:00:00.log.gz", "open_ssl.01:00:00-02:00:00.log",
+
+				"conn.22:00:00-23:00:00.log", "conn_red.22:00:00-23:00:00.log", "conn_blue.22:00:00-23:00:00.log.gz", "open_conn.22:00:00-23:00:00.log",
+				"dns.22:00:00-23:00:00.log", "dns_red.22:00:00-23:00:00.log", "dns_blue.22:00:00-23:00:00.log.gz",
+				"http.22:00:00-23:00:00.log", "http_red.22:00:00-23:00:00.log", "http_blue.22:00:00-23:00:00.log.gz", "open_http.22:00:00-23:00:00.log",
+				"ssl.22:00:00-23:00:00.log", "ssl_red.22:00:00-23:00:00.log", "ssl_blue.22:00:00-23:00:00.log.gz", "open_ssl.22:00:00-23:00:00.log",
+
+				"conn.23:00:00-00:00:00.log", "conn_red.23:00:00-00:00:00.log", "conn_blue.23:00:00-00:00:00.log.gz", "open_conn.23:00:00-00:00:00.log",
+				"dns.23:00:00-00:00:00.log", "dns_red.23:00:00-00:00:00.log", "dns_blue.23:00:00-00:00:00.log.gz",
+				"http.23:00:00-00:00:00.log", "http_red.23:00:00-00:00:00.log", "http_blue.23:00:00-00:00:00.log.gz", "open_http.23:00:00-00:00:00.log",
+				"ssl.23:00:00-00:00:00.log", "ssl_red.23:00:00-00:00:00.log", "ssl_blue.23:00:00-00:00:00.log.gz", "open_ssl.23:00:00-00:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/conn.00:00:00-01:00:00.log", "/logs/conn_blue.00:00:00-01:00:00.log.gz", "/logs/conn_red.00:00:00-01:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.00:00:00-01:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.00:00:00-01:00:00.log", "/logs/dns_blue.00:00:00-01:00:00.log.gz", "/logs/dns_red.00:00:00-01:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.00:00:00-01:00:00.log", "/logs/http_blue.00:00:00-01:00:00.log.gz", "/logs/http_red.00:00:00-01:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.00:00:00-01:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.00:00:00-01:00:00.log", "/logs/ssl_blue.00:00:00-01:00:00.log.gz", "/logs/ssl_red.00:00:00-01:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.00:00:00-01:00:00.log"},
+					},
+					1: {
+						importer.ConnPrefix:     []string{"/logs/conn.01:00:00-02:00:00.log", "/logs/conn_blue.01:00:00-02:00:00.log.gz", "/logs/conn_red.01:00:00-02:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.01:00:00-02:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.01:00:00-02:00:00.log", "/logs/dns_blue.01:00:00-02:00:00.log.gz", "/logs/dns_red.01:00:00-02:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.01:00:00-02:00:00.log", "/logs/http_blue.01:00:00-02:00:00.log.gz", "/logs/http_red.01:00:00-02:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.01:00:00-02:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.01:00:00-02:00:00.log", "/logs/ssl_blue.01:00:00-02:00:00.log.gz", "/logs/ssl_red.01:00:00-02:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.01:00:00-02:00:00.log"},
+					},
+					22: {
+						importer.ConnPrefix:     []string{"/logs/conn.22:00:00-23:00:00.log", "/logs/conn_blue.22:00:00-23:00:00.log.gz", "/logs/conn_red.22:00:00-23:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.22:00:00-23:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.22:00:00-23:00:00.log", "/logs/dns_blue.22:00:00-23:00:00.log.gz", "/logs/dns_red.22:00:00-23:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.22:00:00-23:00:00.log", "/logs/http_blue.22:00:00-23:00:00.log.gz", "/logs/http_red.22:00:00-23:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.22:00:00-23:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.22:00:00-23:00:00.log", "/logs/ssl_blue.22:00:00-23:00:00.log.gz", "/logs/ssl_red.22:00:00-23:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.22:00:00-23:00:00.log"},
+					},
+					23: {
+						importer.ConnPrefix:     []string{"/logs/conn.23:00:00-00:00:00.log", "/logs/conn_blue.23:00:00-00:00:00.log.gz", "/logs/conn_red.23:00:00-00:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.23:00:00-00:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.23:00:00-00:00:00.log", "/logs/dns_blue.23:00:00-00:00:00.log.gz", "/logs/dns_red.23:00:00-00:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.23:00:00-00:00:00.log", "/logs/http_blue.23:00:00-00:00:00.log.gz", "/logs/http_red.23:00:00-00:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.23:00:00-00:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.23:00:00-00:00:00.log", "/logs/ssl_blue.23:00:00-00:00:00.log.gz", "/logs/ssl_red.23:00:00-00:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.23:00:00-00:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "Hour Logs, Containing all Log Types",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				// all logs
+				"conn.01:00:00-02:00:00.log", "open_conn.01:00:00-02:00:00.log", "dns.01:00:00-02:00:00.log", "http.01:00:00-02:00:00.log", "open_http.01:00:00-02:00:00.log", "ssl.01:00:00-02:00:00.log", "open_ssl.01:00:00-02:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					1: {
+						importer.ConnPrefix:     []string{"/logs/conn.01:00:00-02:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.01:00:00-02:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.01:00:00-02:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.01:00:00-02:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.01:00:00-02:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.01:00:00-02:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.01:00:00-02:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "Hour Logs, Missing conn & open_conn Logs",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				// missing conn and open conn
+				"dns.00:00:00-01:00:00.log", "http.00:00:00-01:00:00.log", "open_http.00:00:00-01:00:00.log", "ssl.00:00:00-01:00:00.log", "open_ssl.00:00:00-01:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.DNSPrefix: []string{"/logs/dns.00:00:00-01:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+
+		{
+			name:                 "Hour Logs, Missing conn, has open_conn",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				// missing conn, has open conn
+				"open_conn.02:00:00-03:00:00.log", "dns.02:00:00-03:00:00.log", "http.02:00:00-03:00:00.log", "open_http.02:00:00-03:00:00.log", "ssl.02:00:00-03:00:00.log", "open_ssl.02:00:00-03:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					2: {
+						"open_conn": []string{"/logs/open_conn.02:00:00-03:00:00.log"},
+						"dns":       []string{"/logs/dns.02:00:00-03:00:00.log"},
+						"open_http": []string{"/logs/open_http.02:00:00-03:00:00.log"},
+						"open_ssl":  []string{"/logs/open_ssl.02:00:00-03:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "Hour Logs, Missing open_conn, has conn",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				// missing open conn, has conn
+				"conn.22:00:00-23:00:00.log", "dns.22:00:00-23:00:00.log", "http.22:00:00-23:00:00.log", "ssl.22:00:00-23:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					22: {
+						importer.ConnPrefix: []string{"/logs/conn.22:00:00-23:00:00.log"},
+						importer.DNSPrefix:  []string{"/logs/dns.22:00:00-23:00:00.log"},
+						importer.HTTPPrefix: []string{"/logs/http.22:00:00-23:00:00.log"},
+						importer.SSLPrefix:  []string{"/logs/ssl.22:00:00-23:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "Hour Logs, Missing open_conn, conn, and dns",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				// missing open conn, conn, and dns
+				"http.23:00:00-00:00:00.log", "ssl.23:00:00-00:00:00.log", "open_http.23:00:00-00:00:00.log", "open_ssl.23:00:00-00:00:00.log",
+			},
+			expectedWalkErrors: nil,
+			expectedError:      cmd.ErrNoValidFilesFound, // TODO: error for missing conn, open_conn, and dns
+		},
+		{
+			name:                 "SubDirectories - Non-Hour Logs",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			// subdirectories:       []string{"/sensor1", "/sensor2"},
+			files: []string{
+				"sensor1/conn.log", "sensor1/dns.log", "sensor1/http.log", "sensor1/ssl.log", "sensor1/open_conn.log", "sensor1/open_http.log", "sensor1/open_ssl.log",
+				"sensor2/conn.log", "sensor2/dns.log", "sensor2/http.log", "sensor2/ssl.log", "sensor2/open_conn.log", "sensor2/open_http.log", "sensor2/open_ssl.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/sensor1/conn.log", "/logs/sensor2/conn.log"},
+						importer.OpenConnPrefix: []string{"/logs/sensor1/open_conn.log", "/logs/sensor2/open_conn.log"},
+						importer.DNSPrefix:      []string{"/logs/sensor1/dns.log", "/logs/sensor2/dns.log"},
+						importer.HTTPPrefix:     []string{"/logs/sensor1/http.log", "/logs/sensor2/http.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/sensor1/open_http.log", "/logs/sensor2/open_http.log"},
+						importer.SSLPrefix:      []string{"/logs/sensor1/ssl.log", "/logs/sensor2/ssl.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/sensor1/open_ssl.log", "/logs/sensor2/open_ssl.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "SubDirectories - Hour Logs",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			subdirectories:       []string{"/sensor1", "/sensor2"},
+			files: []string{
+				"sensor1/conn.00:00:00-01:00:00.log", "sensor1/open_conn.00:00:00-01:00:00.log", "sensor1/dns.00:00:00-01:00:00.log", "sensor1/http.00:00:00-01:00:00.log", "sensor1/open_http.00:00:00-01:00:00.log", "sensor1/ssl.00:00:00-01:00:00.log", "sensor1/open_ssl.00:00:00-01:00:00.log",
+				"sensor2/conn.00:00:00-01:00:00.log", "sensor2/open_conn.00:00:00-01:00:00.log", "sensor2/dns.00:00:00-01:00:00.log", "sensor2/http.00:00:00-01:00:00.log", "sensor2/open_http.00:00:00-01:00:00.log", "sensor2/ssl.00:00:00-01:00:00.log", "sensor2/open_ssl.00:00:00-01:00:00.log",
+				"sensor1/conn.23:00:00-00:00:00.log", "sensor1/open_conn.23:00:00-00:00:00.log", "sensor1/dns.23:00:00-00:00:00.log", "sensor1/http.23:00:00-00:00:00.log", "sensor1/open_http.23:00:00-00:00:00.log", "sensor1/ssl.23:00:00-00:00:00.log", "sensor1/open_ssl.23:00:00-00:00:00.log",
+				"sensor2/conn.23:00:00-00:00:00.log", "sensor2/open_conn.23:00:00-00:00:00.log", "sensor2/dns.23:00:00-00:00:00.log", "sensor2/http.23:00:00-00:00:00.log", "sensor2/open_http.23:00:00-00:00:00.log", "sensor2/ssl.23:00:00-00:00:00.log", "sensor2/open_ssl.23:00:00-00:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/sensor1/conn.00:00:00-01:00:00.log", "/logs/sensor2/conn.00:00:00-01:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/sensor1/open_conn.00:00:00-01:00:00.log", "/logs/sensor2/open_conn.00:00:00-01:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/sensor1/dns.00:00:00-01:00:00.log", "/logs/sensor2/dns.00:00:00-01:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/sensor1/http.00:00:00-01:00:00.log", "/logs/sensor2/http.00:00:00-01:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/sensor1/open_http.00:00:00-01:00:00.log", "/logs/sensor2/open_http.00:00:00-01:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/sensor1/ssl.00:00:00-01:00:00.log", "/logs/sensor2/ssl.00:00:00-01:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/sensor1/open_ssl.00:00:00-01:00:00.log", "/logs/sensor2/open_ssl.00:00:00-01:00:00.log"},
+					},
+					23: {
+						importer.ConnPrefix:     []string{"/logs/sensor1/conn.23:00:00-00:00:00.log", "/logs/sensor2/conn.23:00:00-00:00:00.log"},
+						importer.OpenConnPrefix: []string{"/logs/sensor1/open_conn.23:00:00-00:00:00.log", "/logs/sensor2/open_conn.23:00:00-00:00:00.log"},
+						importer.DNSPrefix:      []string{"/logs/sensor1/dns.23:00:00-00:00:00.log", "/logs/sensor2/dns.23:00:00-00:00:00.log"},
+						importer.HTTPPrefix:     []string{"/logs/sensor1/http.23:00:00-00:00:00.log", "/logs/sensor2/http.23:00:00-00:00:00.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/sensor1/open_http.23:00:00-00:00:00.log", "/logs/sensor2/open_http.23:00:00-00:00:00.log"},
+						importer.SSLPrefix:      []string{"/logs/sensor1/ssl.23:00:00-00:00:00.log", "/logs/sensor2/ssl.23:00:00-00:00:00.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/sensor1/open_ssl.23:00:00-00:00:00.log", "/logs/sensor2/open_ssl.23:00:00-00:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "SubDirectories - Multi-Day Logs",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			// subdirectories:       []string{"/2024-04-29", "/2024-05-01"},
+			files: []string{
+				"2024-04-29/conn.log", "2024-04-29/dns.log", "2024-04-29/http.log", "2024-04-29/ssl.log", "2024-04-29/open_conn.log", "2024-04-29/open_http.log", "2024-04-29/open_ssl.log",
+				"2024-05-01/conn.log", "2024-05-01/dns.log", "2024-05-01/http.log", "2024-05-01/ssl.log", "2024-05-01/open_conn.log", "2024-05-01/open_http.log", "2024-05-01/open_ssl.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/2024-04-29/conn.log"},
+						importer.OpenConnPrefix: []string{"/logs/2024-04-29/open_conn.log"},
+						importer.DNSPrefix:      []string{"/logs/2024-04-29/dns.log"},
+						importer.HTTPPrefix:     []string{"/logs/2024-04-29/http.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/2024-04-29/open_http.log"},
+						importer.SSLPrefix:      []string{"/logs/2024-04-29/ssl.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/2024-04-29/open_ssl.log"},
+					},
+				},
+				1: {
+					0: {
+						importer.ConnPrefix:     []string{"/logs/2024-05-01/conn.log"},
+						importer.OpenConnPrefix: []string{"/logs/2024-05-01/open_conn.log"},
+						importer.DNSPrefix:      []string{"/logs/2024-05-01/dns.log"},
+						importer.HTTPPrefix:     []string{"/logs/2024-05-01/http.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/2024-05-01/open_http.log"},
+						importer.SSLPrefix:      []string{"/logs/2024-05-01/ssl.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/2024-05-01/open_ssl.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+
+		{
+			name:                 "Non-Rolling, No Subdirectories",
+			directory:            "/logs",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files: []string{
+				"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					{
+						importer.ConnPrefix:     []string{"/logs/conn.log"},
+						importer.OpenConnPrefix: []string{"/logs/open_conn.log"},
+						importer.DNSPrefix:      []string{"/logs/dns.log"},
+						importer.HTTPPrefix:     []string{"/logs/http.log"},
+						importer.OpenHTTPPrefix: []string{"/logs/open_http.log"},
+						importer.SSLPrefix:      []string{"/logs/ssl.log"},
+						importer.OpenSSLPrefix:  []string{"/logs/open_ssl.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "All sorts of stuff",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			directory:            "/logs",
+			// subdirectories:       []string{"/sensor1", "/2024-05-01"},
+			files: []string{
+				"2024-05-01/conn.log",
+				"2024-05-01/dns.03:00:00-04:00:00.log",
+				"dns.log",
+				"dns.09:00:00-10:00:00.log",
+				"sensor1/ssl.log",
+				"sensor1/conn.log",
+				"sensor1/2025-06-29/conn.04:00:00-05:00:00.log",
+				"sensor1/2025-06-29/http.04:00:00-05:00:00.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+
+				0: {
+					0: {
+						importer.DNSPrefix:  []string{"/logs/dns.log"},
+						importer.SSLPrefix:  []string{"/logs/sensor1/ssl.log"},
+						importer.ConnPrefix: []string{"/logs/sensor1/conn.log"},
+					},
+					9: {importer.DNSPrefix: []string{"/logs/dns.09:00:00-10:00:00.log"}},
+				},
+				1: {
+					0: {importer.ConnPrefix: []string{"/logs/2024-05-01/conn.log"}},
+					3: {importer.DNSPrefix: []string{"/logs/2024-05-01/dns.03:00:00-04:00:00.log"}},
+				},
+				2: {
+					4: {
+						importer.ConnPrefix: []string{"/logs/sensor1/2025-06-29/conn.04:00:00-05:00:00.log"},
+						importer.HTTPPrefix: []string{"/logs/sensor1/2025-06-29/http.04:00:00-05:00:00.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+		{
+			name:                 "Single File Passed In as Root Directory",
+			directoryPermissions: os.FileMode(0o775),
+			filePermissions:      os.FileMode(0o775),
+			files:                []string{"open_conn.log"},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.OpenConnPrefix: []string{"open_conn.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: nil,
+			expectedError:      nil,
+		},
+
+		{
+			name:                 "Duplicate Logs - Same Name, One Newer",
+			directory:            "/logs_dupe",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			files: []string{
+				"conn.log", "conn.log.gz",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix: []string{"/logs_dupe/conn.log.gz"},
+					},
+				},
+			}),
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs_dupe/conn.log", Error: cmd.ErrSkippedDuplicateLog},
+			},
+			expectedError: nil,
+		},
+		{
+			// checks the default case of the switch statement since the test above will be caught by the second case
+			name:                 "Duplicate Logs - Same Name, One Newer - .log.gz File is Older",
+			directory:            "/logs_dupe",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			files: []string{
+				"conn.log.gz", "conn.log",
+			},
+			expectedFiles: createExpectedResults([]cmd.HourlyZeekLogs{
+				0: {
+					0: {
+						importer.ConnPrefix: []string{"/logs_dupe/conn.log"},
+					},
+				},
+			}),
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs_dupe/conn.log.gz", Error: cmd.ErrSkippedDuplicateLog},
+			},
+			expectedError: nil,
+		},
+		{
+			name:                 "No Prefix on Files",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			files: []string{
+				".log.gz", ".log", ".foo",
+			},
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs/.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/.log.gz", Error: cmd.ErrSkippedDuplicateLog},
+				{Path: "/logs/.foo", Error: cmd.ErrIncompatibleFileExtension},
+			},
+			expectedError: cmd.ErrNoValidFilesFound,
+		},
+		{
+			name:                 "Incompatible or Missing File Extensions",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			files: []string{
+				"conn", "dns", "http", "ssl", "open_conn", "open_http", "open_ssl", "conn.00:00:00-01:00:00",
+				".conn", ".conn_", ".dns", ".dns_", ".http", ".http_", ".ssl", ".ssl_", ".bing", "._bong",
+				"dns_file",
+			},
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs/conn", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/dns", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/http", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/ssl", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/open_conn", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/open_http", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/open_ssl", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/conn.00:00:00-01:00:00", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.conn", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.conn_", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.dns", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.dns_", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.http", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.http_", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.ssl", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.ssl_", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/.bing", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/._bong", Error: cmd.ErrIncompatibleFileExtension},
+				{Path: "/logs/dns_file", Error: cmd.ErrIncompatibleFileExtension},
+			},
+			expectedError: cmd.ErrNoValidFilesFound,
+		},
+		{
+			name:                 "Invalid Log Types",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			files: []string{
+				"files.log", "ntp.log", "radius.log", "sip.log", "x509.log.gz", "dhcp.log", "weird.log",
+				"conn_summary.log", "conn-summary.log", "foo.log",
+			},
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs/files.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/ntp.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/radius.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/sip.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/x509.log.gz", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/dhcp.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/weird.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/conn_summary.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/conn-summary.log", Error: cmd.ErrInvalidLogType},
+				{Path: "/logs/foo.log", Error: cmd.ErrInvalidLogType},
+			},
+			expectedError: cmd.ErrNoValidFilesFound,
+		},
+		{
+			name:                 "No Read Permissions on Files",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o000),
+			files: []string{
+				"conn.log", "dns.log", "http.log", "ssl.log", "open_conn.log", "open_http.log", "open_ssl.log",
+			},
+			expectedWalkErrors: []cmd.WalkError{
+				{Path: "/logs/conn.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/dns.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/http.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/ssl.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/open_conn.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/open_http.log", Error: cmd.ErrInsufficientReadPermissions},
+				{Path: "/logs/open_ssl.log", Error: cmd.ErrInsufficientReadPermissions},
+			},
+			expectedError: cmd.ErrNoValidFilesFound,
+		},
+		{
+			name:                 "No Files, Only SubDirectories",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			subdirectories:       []string{"/sensor1", "/sensor2"},
+			expectedWalkErrors:   nil,
+			expectedError:        cmd.ErrNoValidFilesFound,
+		},
+		{
+			name:                 "No Files",
+			directory:            "/logs",
+			directoryPermissions: iofs.FileMode(0o775),
+			filePermissions:      iofs.FileMode(0o775),
+			expectedWalkErrors:   nil,
+			expectedError:        util.ErrDirIsEmpty,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			// Create the directory
+			if test.directory != "" {
+				err := afs.MkdirAll(test.directory, test.directoryPermissions)
+				require.NoError(t, err)
+			}
+
+			for _, subdirectory := range test.subdirectories {
+				err := afs.MkdirAll(filepath.Join(test.directory, subdirectory), test.directoryPermissions)
+				require.NoError(t, err)
+			}
+
+			// create the files
+			for i, file := range test.files {
+
+				// if the test is for duplicate logs, wait a little bit to simulate a newer last modified time
+				if strings.HasPrefix(test.name, "Duplicate Logs - Same Name, One Newer") {
+					if i > 0 {
+						time.Sleep(300 * time.Millisecond)
+					}
+				}
+
+				err := afero.WriteFile(afs, filepath.Join(test.directory, file), []byte("testytesttestboop"), test.filePermissions)
+				require.NoError(t, err, "creating mock file should not produce an error")
+			}
+
+			// walk the directory
+			var logMap []cmd.HourlyZeekLogs
+			var walkErrors []cmd.WalkError
+			var err error
+
+			// since some of the tests are for files passed in to the import command instead of the root directory, we need to
+			// simulate that accordingly
+			if test.directory != "" {
+				logMap, walkErrors, err = cmd.WalkFiles(afs, test.directory)
+			} else {
+				logMap, walkErrors, err = cmd.WalkFiles(afs, strings.Join(test.files, " "))
+			}
+
+			// check if the error is expected
+			if test.expectedError == nil {
+				require.NoError(t, err, "running WalkFiles should not produce an error")
+			} else {
+				require.ErrorIs(t, err, test.expectedError, "error should match expected value")
+
+			}
+
+			// verify that the returned log map matches the expected values
+			require.Equal(t, test.expectedFiles, logMap, "log map should match expected value")
+
+			// check if elements match for walk errors instead of equal so that we don't have to worry about
+			// the errors being in the right order
+
+			// verify that the returned walk errors match the expected values
+			require.ElementsMatch(t, test.expectedWalkErrors, walkErrors, "walk errors should match expected value")
+
+			// clean up the directory
+			err = afs.RemoveAll(test.directory)
+			require.NoError(t, err, "removing mock directory should not produce an error")
+		})
+
+	}
+}
+
+func TestParseHourFromFilename(t *testing.T) {
+	tests := []struct {
+		name     string
+		filename string
+		wantHour int
+		wantErr  error
+	}{
+		{
+			name:     "Simple Log with No Hour Segment",
+			filename: "conn.log",
+			wantHour: 0,
+			wantErr:  nil,
+		},
+
+		{
+			name:     "Valid hour middle range",
+			filename: "log.15:30",
+			wantHour: 15,
+			wantErr:  nil,
+		},
+		{
+			name:     "Valid hour lower bound",
+			filename: "log.00:00",
+			wantHour: 0,
+			wantErr:  nil,
+		},
+		{
+			name:     "Valid hour upper bound",
+			filename: "log.23:59",
+			wantHour: 23,
+			wantErr:  nil,
+		},
+		{
+			name:     "Invalid Hour Range",
+			filename: "log.24:00",
+			wantHour: 0,
+			wantErr:  cmd.ErrInvalidLogHourRange,
+		},
+		{
+			name:     "Non-numeric Hour Segment",
+			filename: "log.ab:cd",
+			wantHour: 0,
+			wantErr:  cmd.ErrInvalidLogHourFormat,
+		},
+		{
+			name:     "Incomplete Hour Segment",
+			filename: "log.:34",
+			wantHour: 0,
+			wantErr:  cmd.ErrInvalidLogHourFormat,
+		},
+		{
+			name:     "Extra characters",
+			filename: "log.12x:34",
+			wantHour: 0,
+			wantErr:  cmd.ErrInvalidLogHourFormat,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotHour, err := cmd.ParseHourFromFilename(test.filename)
+			require.Equal(t, test.wantErr, err, "expected error to be %v, got %v", test.wantErr, err)
+			require.Equal(t, test.wantHour, gotHour, "expected hour to be %v, got %v", test.wantHour, gotHour)
+		})
+	}
+}
+
+func TestValidateDatabaseName(t *testing.T) {
+	type testCase struct {
+		name      string
+		db        string
+		shouldErr bool
+	}
+
+	tests := []testCase{
+		{name: "All alpha characters", db: "vsagent"},
+		{name: "All alphanumeric characters", db: "dnscat20"},
+		{name: "Starting with a capital letter", db: "Vsagent", shouldErr: true},
+		{name: "All caps", db: "INFORMATION_SCHEMA", shouldErr: true},
+		{name: "Starting with an underscore", db: "_vsagent", shouldErr: true},
+		{name: "Name is reserved: default", db: "default", shouldErr: true},
+		{name: "Name is reserved: system", db: "system", shouldErr: true},
+		{name: "Name is reserved: information_schema", db: "information_schema", shouldErr: true},
+		{name: "Name is reserved: metadatabase", db: "metadatabase", shouldErr: true},
+		{name: "Contains special characters", db: "ch!ck3n$tr!p", shouldErr: true},
+		{name: "Contains a hyphen", db: "combined__0000-rolling", shouldErr: true},
+		{name: "Ends with underscore", db: "dnscat2_", shouldErr: true},
+		{name: "Common name, dnscat2_ja3_strobe", db: "dnscat2_ja3_strobe"},
+		{name: "Common name, combined__0000_rolling", db: "combined__0000_rolling"},
+		{name: "Common name, seconion_2024_05_15", db: "combined__0000_rolling"},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := cmd.ValidateDatabaseName(test.db)
+			require.Equal(t, test.shouldErr, err != nil, "expected error:%t, got error: %t", test.shouldErr, err)
+		})
+	}
+}

cmd/list.go

@@ -0,0 +1,102 @@
+package cmd
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/charmbracelet/lipgloss"
+	"github.com/charmbracelet/lipgloss/table"
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ListCommand = &cli.Command{
+	Name:        "list",
+	Usage:       "list available datasets",
+	UsageText:   "list",
+	Description: "lists available datasets",
+	Args:        false,
+	Flags: []cli.Flag{
+		ConfigFlag(false),
+	},
+	Action: func(cCtx *cli.Context) error {
+
+		// check if too many arguments were provided
+		if cCtx.NArg() > 0 {
+			return ErrTooManyArguments
+		}
+
+		// set up file system interface
+		afs := afero.NewOsFs()
+
+		// run the delete command
+		if err := runListCmd(afs, cCtx.String("config")); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+func runListCmd(afs afero.Fs, configPath string) error {
+
+	cfg, err := config.LoadConfig(afs, configPath)
+	if err != nil {
+		return err
+	}
+
+	// connect to server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	if err != nil {
+		return err
+	}
+
+	dbs, err := server.ListImportDatabases()
+	if err != nil {
+		return err
+	}
+
+	if len(dbs) == 0 {
+		fmt.Println("No available datasets.")
+	}
+
+	t := FormatListTable(dbs)
+	fmt.Println(t)
+	return nil
+}
+
+func FormatListTable(dbs []database.ImportDatabase) *table.Table {
+	var data [][]string
+
+	for _, d := range dbs {
+		data = append(data, []string{d.Name, strconv.FormatBool(d.Rolling), fmt.Sprintf("%s - %s", d.MinTS.Format("2006-01-02 15:04"), d.MaxTS.Format("2006-01-02 15:04"))})
+	}
+
+	re := lipgloss.NewRenderer(os.Stdout)
+	baseStyle := re.NewStyle().Padding(0, 1)
+	headerStyle := baseStyle.Foreground(lipgloss.Color("252")).Bold(true)
+
+	headers := []string{"Name", "Rolling", "Time Range (UTC)"}
+	t := table.New().
+		Border(lipgloss.NormalBorder()).
+		BorderStyle(re.NewStyle().Foreground(lipgloss.Color("238"))).
+		Headers(headers...).
+		Rows(data...).
+		StyleFunc(func(row, col int) lipgloss.Style {
+			if row == 0 {
+				return headerStyle
+			}
+
+			even := row%2 == 0
+
+			if even {
+				return baseStyle.Foreground(lipgloss.Color("245"))
+			}
+			return baseStyle.Foreground(lipgloss.Color("252"))
+		})
+	return t
+}

cmd/list_test.go

@@ -0,0 +1,68 @@
+package cmd_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func (c *CmdTestSuite) TestFormatListTable() {
+	require := require.New(c.T())
+
+	_, err := cmd.RunImportCmd(time.Now(), c.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "dnscat", false, true)
+	require.NoError(err, "importing dnscat data should not produce an error")
+
+	_, err = cmd.RunImportCmd(time.Now(), c.cfg, afero.NewOsFs(), "../test_data/proxy", "proxy", false, true)
+	require.NoError(err, "importing proxy data should not produce an error")
+
+	_, dropletSubnet, err := net.ParseCIDR("64.225.56.201/32")
+	require.NoError(err)
+	c.cfg.Filter.InternalSubnets = append(c.cfg.Filter.InternalSubnets, dropletSubnet)
+	c.cfg.Filter.FilterExternalToInternal = false
+	err = config.UpdateConfig(c.cfg)
+	require.NoError(err, "config should update without error")
+
+	_, err = cmd.RunImportCmd(time.Now(), c.cfg, afero.NewOsFs(), "../test_data/missing_host/2024-04-19", "fake_rolling", true, true)
+	require.NoError(err, "importing fake_rolling data should not produce an error")
+
+	_, err = cmd.RunImportCmd(time.Now(), c.cfg, afero.NewOsFs(), "../test_data/missing_host/2024-04-20", "fake_rolling", true, false)
+	require.NoError(err, "importing 2nd fake_rolling data should not produce an error")
+
+	// connect to server
+	server, err := database.ConnectToServer(context.Background(), c.cfg)
+	require.NoError(err)
+
+	dbs, err := server.ListImportDatabases()
+	require.NoError(err)
+
+	output := cmd.FormatListTable(dbs)
+
+	lines := strings.Split(output.String(), "\n")
+	require.Len(lines, 7)
+	lines = lines[3:6]
+
+	expectedDBs := []struct {
+		name    string
+		rolling string
+		tsRange string
+	}{
+		{name: "fake_rolling", rolling: "true", tsRange: "2024-04-18 20:07 - 2024-04-20 23:59"},
+		{name: "proxy", rolling: "false", tsRange: "2022-12-22 18:48 - 2023-01-05 18:48"},
+		{name: "dnscat", rolling: "false", tsRange: "2018-01-30 18:00 - 2018-01-31 18:14"},
+	}
+	for i, line := range lines {
+		cols := strings.Split(line, "│")
+		require.Len(cols, 5)
+		cols = cols[1:4]
+		require.Equal(expectedDBs[i].name, strings.TrimSpace(cols[0]))
+		require.Equal(expectedDBs[i].rolling, strings.TrimSpace(cols[1]))
+		require.Equal(expectedDBs[i].tsRange, strings.TrimSpace(cols[2]))
+	}
+}

cmd/validate.go

@@ -0,0 +1,79 @@
+package cmd
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/util"
+	"errors"
+	"fmt"
+
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ErrInvalidConfig = errors.New("encountered invalid configuration values")
+
+var ValidateConfigCommand = &cli.Command{
+	Name:      "validate",
+	Usage:     "validate a configuration file",
+	UsageText: "validate [--config FILE]",
+	Args:      false,
+	Flags: []cli.Flag{
+		ConfigFlag(false),
+	},
+	Action: func(cCtx *cli.Context) error {
+		// check if a config was provided and is not empty
+		if cCtx.String("config") == "" {
+			return ErrMissingConfigPath
+		}
+
+		// check if too many arguments were provided
+		if cCtx.NArg() > 0 {
+			return ErrTooManyArguments
+		}
+
+		afs := afero.NewOsFs()
+
+		// validate config file
+		if err := RunValidateConfigCommand(afs, cCtx.String("config")); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+func RunValidateConfigCommand(afs afero.Fs, configPath string) error {
+	// validate config file path
+	if err := ValidateConfigPath(afs, configPath); err != nil {
+		return err
+	}
+
+	// load config path
+	_, err := config.LoadConfig(afs, configPath)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("configuration file is valid")
+
+	return nil
+}
+
+func ValidateConfigPath(afs afero.Fs, configPath string) error {
+	if configPath == "" {
+		return ErrMissingConfigPath
+	}
+
+	// get relative file path
+	_, err := util.ParseRelativePath(configPath)
+	if err != nil {
+		return err
+	}
+
+	// validate file path
+	if err := util.ValidateFile(afs, configPath); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/view.go

@@ -0,0 +1,141 @@
+package cmd
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/util"
+	"activecm/rita/viewer"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/spf13/afero"
+	"github.com/urfave/cli/v2"
+)
+
+var ErrMissingSearchValue = errors.New("search value cannot be empty")
+var ErrMissingSearchStdout = errors.New("cannot apply search without --stdout")
+var ErrMissingLimitStdout = errors.New("cannot apply limit without --stdout")
+var ErrInvalidViewLimit = errors.New("limit must be a positive interger greater than 0")
+var ErrDatabaseNotFound = errors.New("database not found")
+
+var ViewCommand = &cli.Command{
+	Name:  "view",
+	Usage: "view <dataset name>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:     "stdout",
+			Aliases:  []string{"o"},
+			Usage:    "pipe comma-delimited data to stdout",
+			Required: false,
+		},
+		&cli.StringFlag{
+			Name:     "search",
+			Aliases:  []string{"s"},
+			Usage:    `search criteria to apply to results piped to stdout, only works with --stdout/-o flag, format: -s="field:value, field:value, ..."`,
+			Required: false,
+		},
+		&cli.IntFlag{
+			Name:     "limit",
+			Aliases:  []string{"l"},
+			Usage:    "limit the number of results to display",
+			Required: false,
+		},
+	},
+	Action: func(cCtx *cli.Context) error {
+		afs := afero.NewOsFs()
+
+		// flags must go before the argument, otherwise they won't be applied ._.
+		// we can either make the db name a flag or see if cobra is any better
+		if !cCtx.Args().Present() {
+			return ErrMissingDatabaseName
+		}
+
+		if err := ValidateDatabaseName(cCtx.Args().First()); err != nil {
+			return err
+		}
+
+		if cCtx.IsSet("search") {
+			if !cCtx.Bool("stdout") {
+				return ErrMissingSearchStdout
+			}
+
+			if cCtx.String("search") == "" {
+				return ErrMissingSearchValue
+			}
+		}
+
+		// validate limit flag
+		if cCtx.IsSet("limit") {
+			if !cCtx.Bool("stdout") {
+				return ErrMissingLimitStdout
+			}
+
+			if cCtx.Int("limit") <= 0 {
+				return ErrInvalidViewLimit
+			}
+		}
+
+		err := runViewCmd(afs, cCtx.String("config"), cCtx.Args().First(), cCtx.Bool("stdout"), cCtx.String("search"), cCtx.Int("limit"))
+		return err
+	},
+}
+
+func runViewCmd(afs afero.Fs, configPath string, dbName string, stdout bool, search string, limit int) error {
+	fmt.Printf("Viewing database: %s\n", dbName)
+
+	// load config file
+	cfg, err := config.LoadConfig(afs, configPath)
+	if err != nil {
+		return err
+	}
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), dbName, cfg, nil)
+	if err != nil {
+		return err
+	}
+
+	// determine which max timestamp to use for relative time calculations
+	minTimestamp, maxTimestamp, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return ErrDatabaseNotFound
+		}
+		return err
+	}
+
+	// if stdout was requested, get CSV output
+	if stdout {
+
+		// get CSV output
+		csvData, err := viewer.GetCSVOutput(db, minTimestamp, util.GetRelativeFirstSeenTimestamp(useCurrentTime, maxTimestamp), search, limit)
+		if err != nil {
+			return err
+		}
+
+		// print CSV data to stdout
+		fmt.Println(csvData)
+
+	} else {
+
+		// create UI
+		if err := viewer.CreateUI(cfg, db, useCurrentTime, maxTimestamp, minTimestamp); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// func validateDatabaseName(dbName string) error {
+// 	//  do not allow anything but alphanumeric and underscores for the database name
+// 	re := regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
+
+// 	if !re.MatchString(dbName) {
+// 		return fmt.Errorf("invalid database name: %s", dbName)
+// 	}
+
+// 	return nil
+// }

config.hjson

@@ -0,0 +1,12 @@
+{
+    update_check_enabled: true,
+    filtering: {
+        filter_external_to_internal: true,
+        internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"]
+    },
+    threat_intel: {
+        online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+        custom_feeds_directory: "./deployment/threat_intel_feeds"
+    },
+    http_extensions_file_path: "./deployment/http_extensions_list.csv"
+}

config/config.go

@@ -0,0 +1,656 @@
+package config
+
+import (
+	"activecm/rita/util"
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/hjson/hjson-go/v4"
+	"github.com/spf13/afero"
+)
+
+var appConfig *Config
+var loadedConfig bool
+var once sync.Once
+
+var Version string
+
+const DefaultConfigPath = "./config.hjson"
+
+var errInvalidImpactCategory = errors.New("invalid impact category: must be 'critical', 'high', 'medium', 'low', or 'none'")
+
+const (
+	NONE_CATEGORY_SCORE   = 0.2
+	LOW_CATEGORY_SCORE    = 0.4
+	MEDIUM_CATEGORY_SCORE = 0.6
+	HIGH_CATEGORY_SCORE   = 0.8
+
+	CriticalThreat ImpactCategory = "critical"
+	HighThreat     ImpactCategory = "high"
+	MediumThreat   ImpactCategory = "medium"
+	LowThreat      ImpactCategory = "low"
+	NoneThreat     ImpactCategory = "none"
+)
+
+type (
+	ThreatIntel struct {
+		OnlineFeeds          []string `json:"online_feeds"`
+		CustomFeedsDirectory string   `json:"custom_feeds_directory"`
+	}
+
+	// ScoreThresholds is used for indicators that have prorated (graduated) values rather than
+	// binary outcomes. This allows for the definition of the severity of an indicator by categorizing
+	// it into one of several buckets (Base, Low, Med, High), each representing a range of values
+	ScoreThresholds struct {
+		Base int `json:"base"`
+		Low  int `json:"low"`
+		Med  int `json:"medium"`
+		High int `json:"high"`
+	}
+
+	ImpactCategory string
+
+	// ScoreImpact is used for indicators that have a binary outcomes but still need to express the
+	// impact of being true on the overall score.
+	ScoreImpact struct {
+		Category ImpactCategory `json:"category"`
+		Score    float32
+	}
+
+	Scoring struct {
+		Beacon Beacon `json:"beacon"`
+
+		LongConnectionMinimumDuration int             `json:"long_connection_minimum_duration"`
+		LongConnectionScoreThresholds ScoreThresholds `json:"long_connection_score_thresholds"`
+
+		C2SubdomainThreshold int             `json:"c2_subdomain_threshold"`
+		C2ScoreThresholds    ScoreThresholds `json:"c2_score_thresholds"`
+
+		StrobeImpact ScoreImpact `json:"strobe_impact"`
+
+		ThreatIntelImpact ScoreImpact `json:"threat_intel_impact"`
+	}
+
+	Modifiers struct {
+		ThreatIntelScoreIncrease     float32 `json:"threat_intel_score_increase"`
+		ThreatIntelDataSizeThreshold int64   `json:"threat_intel_datasize_threshold"`
+
+		PrevalenceScoreIncrease     float32 `json:"prevalence_score_increase"`
+		PrevalenceIncreaseThreshold float32 `json:"prevalence_increase_threshold"`
+		PrevalenceScoreDecrease     float32 `json:"prevalence_score_decrease"`
+		PrevalenceDecreaseThreshold float32 `json:"prevalence_decrease_threshold"`
+
+		FirstSeenScoreIncrease     float32 `json:"first_seen_score_increase"`
+		FirstSeenIncreaseThreshold float32 `json:"first_seen_increase_threshold"`
+		FirstSeenScoreDecrease     float32 `json:"first_seen_score_decrease"`
+		FirstSeenDecreaseThreshold float32 `json:"first_seen_decrease_threshold"`
+
+		MissingHostCountScoreIncrease float32 `json:"missing_host_count_score_increase"`
+
+		RareSignatureScoreIncrease float32 `json:"rare_signature_score_increase"`
+
+		C2OverDNSDirectConnScoreIncrease float32 `json:"c2_over_dns_direct_conn_score_increase"`
+
+		MIMETypeMismatchScoreIncrease float32 `json:"mime_type_mismatch_score_increase"`
+	}
+
+	Beacon struct {
+		UniqueConnectionThreshold       int64           `json:"unique_connection_threshold"`
+		TsWeight                        float64         `json:"timestamp_score_weight"`
+		DsWeight                        float64         `json:"datasize_score_weight"`
+		DurWeight                       float64         `json:"duration_score_weight"`
+		HistWeight                      float64         `json:"histogram_score_weight"`
+		DurMinHours                     int             `json:"duration_min_hours_seen"`
+		DurIdealNumberOfConsistentHours int             `json:"duration_consistency_ideal_hours_seen"`
+		HistModeSensitivity             float64         `json:"histogram_mode_sensitivity"`
+		HistBimodalOutlierRemoval       int             `json:"histogram_bimodal_outlier_removal"`
+		HistBimodalMinHours             int             `json:"histogram_bimodal_min_hours_seen"`
+		ScoreThresholds                 ScoreThresholds `json:"score_thresholds"`
+	}
+
+	Config struct {
+		DBConnection       string // set by .env file
+		UpdateCheckEnabled bool   `json:"update_check_enabled"`
+		Filter             Filter `json:"filtering"`
+
+		HTTPExtensionsFilePath string `json:"http_extensions_file_path"`
+
+		// writer
+		BatchSize             int `json:"batch_size"`
+		MaxQueryExecutionTime int `json:"max_query_execution_time"`
+
+		// historical first seen
+		MonthsToKeepHistoricalFirstSeen int `json:"months_to_keep_historical_first_seen"`
+
+		Scoring Scoring `json:"scoring"`
+
+		Modifiers Modifiers `json:"modifiers"`
+
+		ThreatIntel ThreatIntel `json:"threat_intel"`
+
+		LogLevel       int  `json:"log_level"`
+		LoggingEnabled bool `json:"logging_enabled"`
+	}
+)
+
+// LoadConfig loads the config file from the specified path and returns a valid Config if successful
+func LoadConfig(afs afero.Fs, path string) (*Config, error) {
+	var err error
+	// once.Do(func() {
+	if !loadedConfig {
+		if path == "" {
+			path = DefaultConfigPath
+		}
+		cfg, errRead := ReadFileConfig(afs, path)
+		appConfig = &cfg
+		err = errRead
+		if Version == "" {
+			Version = "devel"
+		}
+		loadedConfig = true
+	}
+	// })
+	return appConfig, err
+}
+
+// GetConfig returns a global Config that must be previously loaded using LoadConfig, returns an error if not loaded
+func GetConfig() (*Config, error) {
+	if !loadedConfig {
+		return appConfig, fmt.Errorf("tried reading from config, but it has not been loaded yet")
+	}
+	return appConfig, nil
+}
+
+// UpdateConfig updates the global config by overriding the existing config with the passed in config. This should only be used for tests.
+func UpdateConfig(cfg *Config) error {
+	if !loadedConfig {
+		return fmt.Errorf("tried reading from config, but it has not been loaded yet")
+	}
+	appConfig = cfg
+	return nil
+}
+
+// ReadFileConfig attempts to read the config file at the specified path and
+// returns a config object, using the default config if the file was unable to be read.
+func ReadFileConfig(afs afero.Fs, path string) (Config, error) {
+	// get the default cfg
+	cfg, err := getDefaultConfig()
+	if err != nil {
+		return cfg, err
+	}
+
+	// read the config file
+	contents, err := readFile(afs, path)
+	if err != nil {
+		return cfg, err
+	}
+
+	// parse file contents
+	err = cfg.parseJSON(contents)
+	if err != nil {
+		return cfg, err
+	}
+
+	// validate values
+	err = cfg.Validate()
+	if err != nil {
+		return cfg, err
+	}
+
+	return cfg, nil
+}
+
+// getDefaultConfig returns a Config object with default values
+func getDefaultConfig() (Config, error) {
+	// set default config values
+	// cfg := defaultConfig
+	cfg := defaultConfig()
+
+	// get the database connection string
+	connection := os.Getenv("DB_ADDRESS")
+	if connection == "" {
+		return Config{}, errors.New("environment variable DB_ADDRESS not set")
+	}
+	cfg.DBConnection = connection
+
+	// set up the filter based on default values
+	// (must be done to convert strings in the default config variable to net.IPNet)
+	err := cfg.parseFilter()
+	if err != nil {
+		return cfg, err
+	}
+
+	return cfg, nil
+}
+
+// readFile reads the config file at the specified path and returns its contents
+func readFile(afs afero.Fs, path string) ([]byte, error) {
+	// validate file
+	err := util.ValidateFile(afs, path)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := afero.ReadFile(afs, path)
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}
+
+// parseJSON parses the JSON config file and stores the result in the provided Config object
+func (cfg *Config) parseJSON(configFile []byte) error {
+	// parse the JSON config file
+	if err := hjson.Unmarshal(configFile, cfg); err != nil {
+		return err
+	}
+
+	// parse the new subnet filter values
+	if err := cfg.parseFilter(); err != nil {
+		return err
+	}
+
+	// parse impact category scores
+	if err := cfg.parseImpactCategoryScores(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ResetConfig resets the config values to default
+func (cfg *Config) ResetConfig() error {
+	newConfig, err := getDefaultConfig()
+	if err != nil {
+		return err
+	}
+	*cfg = newConfig
+	loadedConfig = false
+	return nil
+}
+
+func (cfg *Config) Validate() error {
+	// validate the configured values
+	if err := cfg.verifyConfig(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// verifyConfig validates the configuration settings
+func (cfg *Config) verifyConfig() error {
+	if cfg.DBConnection == "" {
+		return fmt.Errorf("DBConnection cannot be empty")
+	}
+
+	// validate that there is at least one internal subnet, or else we cannot do analysis
+	if len(cfg.Filter.InternalSubnets) < 1 {
+		return fmt.Errorf("the list of internal subnets is empty, got %v", cfg.Filter.InternalSubnets)
+	}
+
+	if len(cfg.HTTPExtensionsFilePath) < 1 {
+		return fmt.Errorf("the valid HTTP extensions file path is not set, got %v", cfg.HTTPExtensionsFilePath)
+	}
+
+	// validate the batch size
+	if cfg.BatchSize < 25000 || cfg.BatchSize > 2000000 {
+		return fmt.Errorf("the batch size for writing to the database must be between 25k and 2 million")
+	}
+
+	// validate the max query execution time
+	if cfg.MaxQueryExecutionTime < 1 || cfg.MaxQueryExecutionTime > 2000000 {
+		return fmt.Errorf("the max database query execution time must be between 1 second and 2 million seconds")
+	}
+
+	// validate historical first seen months
+	if cfg.MonthsToKeepHistoricalFirstSeen < 1 || cfg.MonthsToKeepHistoricalFirstSeen > 60 {
+		return fmt.Errorf("the historical first seen months must be between 1 and 60, got %v", cfg.MonthsToKeepHistoricalFirstSeen)
+	}
+
+	// validate the configured unique connection threshold (need at least 3 intervals, which means at least 4 connections)
+	if cfg.Scoring.Beacon.UniqueConnectionThreshold < 4 {
+		return fmt.Errorf("the unique connection threshold must be at least 4, got %v", cfg.Scoring.Beacon.UniqueConnectionThreshold)
+	}
+
+	// validate the configured score weights
+	totalWeight := 0.0
+	weights := []float64{
+		cfg.Scoring.Beacon.TsWeight,
+		cfg.Scoring.Beacon.DsWeight,
+		cfg.Scoring.Beacon.DurWeight,
+		cfg.Scoring.Beacon.HistWeight,
+	}
+	for _, weight := range weights {
+		if weight < 0 || weight > 1 {
+			return fmt.Errorf("the weight must be between 0 and 1, got %v", weight)
+		}
+		totalWeight += weight
+	}
+
+	// sum of weights must equal 1
+	if totalWeight != 1 {
+		return fmt.Errorf("the sum of the weights must equal 1, got %v", totalWeight)
+	}
+
+	// validate the configured minimum hours seen for duration
+	if cfg.Scoring.Beacon.DurMinHours < 1 {
+		return fmt.Errorf("the minimum hours seen for duration must be at least 1, got %v", cfg.Scoring.Beacon.DurMinHours)
+	}
+
+	// validate the configured ideal number of consistent hours seen
+	if cfg.Scoring.Beacon.DurIdealNumberOfConsistentHours < 1 {
+		return fmt.Errorf("the ideal number of consistent hours seen must be at least 1, got %v", cfg.Scoring.Beacon.DurIdealNumberOfConsistentHours)
+	}
+
+	// validate the configured mode sensitivity
+	if cfg.Scoring.Beacon.HistModeSensitivity < 0 || cfg.Scoring.Beacon.HistModeSensitivity > 1 {
+		return fmt.Errorf("the mode sensitivity must be between 0 and 1, got %v", cfg.Scoring.Beacon.HistModeSensitivity)
+	}
+
+	// validate the configured bimodal outlier removal
+	if cfg.Scoring.Beacon.HistBimodalOutlierRemoval < 0 {
+		return fmt.Errorf("the bimodal outlier removal must be at least 0, got %v", cfg.Scoring.Beacon.HistBimodalOutlierRemoval)
+	}
+
+	// validate the configured min hours seen for histogram
+	// this is to ensure that the bimodal fit score is not calculated for histograms with too few hours, as in that case
+	// a histogram with 1-2 bars will always be given a high bimoal fit score as it technically has 1-2 modes
+	if cfg.Scoring.Beacon.HistBimodalMinHours < 3 {
+		return fmt.Errorf("the minimum hours seen for histogram must be at least 3, got %v", cfg.Scoring.Beacon.HistBimodalMinHours)
+	}
+
+	// validate the configured beacon score thresholds ( scores are between 0 and 100 )
+	if err := validateScoreThresholds(cfg.Scoring.Beacon.ScoreThresholds, 0, 100); err != nil {
+		return err
+	}
+
+	// TODO: i'm pretty sure this and the c2 subdomain min thresh can be replaced with the threshold.base value?
+	// validate the configured long connection minimum duration
+	if cfg.Scoring.LongConnectionMinimumDuration <= 0 {
+		return fmt.Errorf("the long connection minimum duration must be at least greater than 0, got %v", cfg.Scoring.LongConnectionMinimumDuration)
+	}
+
+	// validate the configured long connection score thresholds ( between 0 and 24 hours )
+	if err := validateScoreThresholds(cfg.Scoring.LongConnectionScoreThresholds, 0, 24*3600); err != nil {
+		return err
+	}
+
+	// validate the configured C2 subdomain threshold
+	if cfg.Scoring.C2SubdomainThreshold <= 0 {
+		return fmt.Errorf("the C2 subdomain threshold must be at least greater than 0, got %v", cfg.Scoring.C2SubdomainThreshold)
+	}
+
+	// validate the configured C2 score thresholds ( no max limit )
+	if err := validateScoreThresholds(cfg.Scoring.C2ScoreThresholds, 0, -1); err != nil {
+		return err
+	}
+
+	// validate the configured strobe impact category
+	if err := ValidateImpactCategory(cfg.Scoring.StrobeImpact.Category); err != nil {
+		return err
+	}
+
+	// threat intel struct can be empty, so no need for validation
+
+	// validate the configured threat intel impact category
+	if err := ValidateImpactCategory(cfg.Scoring.ThreatIntelImpact.Category); err != nil {
+		return err
+	}
+
+	// validate the configured threat intel modifier values
+	if cfg.Modifiers.ThreatIntelScoreIncrease < 0 || cfg.Modifiers.ThreatIntelScoreIncrease > 1 {
+		return fmt.Errorf("the threat intel modifier score increase must be between 0 and 1, got %v", cfg.Modifiers.ThreatIntelScoreIncrease)
+	}
+	// validate the configured threat intel modifier data size threshold (must be greater than 0 and less than the max int64 value)
+	if cfg.Modifiers.ThreatIntelDataSizeThreshold < 1 {
+		return fmt.Errorf("the threat intel modifier data size threshold must be greater than 0, got %v", cfg.Modifiers.ThreatIntelScoreIncrease)
+	}
+
+	// validate the configured prevalence score increase modifier values
+	if cfg.Modifiers.PrevalenceScoreIncrease < 0 || cfg.Modifiers.PrevalenceScoreIncrease > 1 {
+		return fmt.Errorf("the prevalence modifier score increase must be between 0 and 1, got %v", cfg.Modifiers.PrevalenceScoreIncrease)
+	}
+	// validate score increase threshold
+	if cfg.Modifiers.PrevalenceIncreaseThreshold < 0 || cfg.Modifiers.PrevalenceIncreaseThreshold > 1 {
+		return fmt.Errorf("the prevalence modifier increase threshold must be between 0 and 1, got %v", cfg.Modifiers.PrevalenceIncreaseThreshold)
+	}
+
+	// validate the configured prevalence score decrease modifier values
+	if cfg.Modifiers.PrevalenceScoreDecrease < 0 || cfg.Modifiers.PrevalenceScoreDecrease > 1 {
+		return fmt.Errorf("the prevalence modifier score decrease must be between 0 and 1, got %v", cfg.Modifiers.PrevalenceScoreDecrease)
+	}
+	// validate score decrease threshold (must be between 0 and 1 and greater than the increase threshold)
+	if cfg.Modifiers.PrevalenceDecreaseThreshold < 0 || cfg.Modifiers.PrevalenceDecreaseThreshold > 1 {
+		return fmt.Errorf("the prevalence modifier decrease threshold must be between 0 and 1, got %v", cfg.Modifiers.PrevalenceDecreaseThreshold)
+	}
+	if cfg.Modifiers.PrevalenceDecreaseThreshold <= cfg.Modifiers.PrevalenceIncreaseThreshold {
+		return fmt.Errorf("the prevalence modifier decrease threshold must be greater than the increase threshold, got %v", cfg.Modifiers.PrevalenceDecreaseThreshold)
+	}
+
+	// validate the configured first seen score increase modifier values (must be between 0 and 1)
+	if cfg.Modifiers.FirstSeenScoreIncrease < 0 || cfg.Modifiers.FirstSeenScoreIncrease > 1 {
+		return fmt.Errorf("the first seen modifier score increase must be between 0 and 1, got %v", cfg.Modifiers.FirstSeenScoreIncrease)
+	}
+	// validate first seen score increase threshold (must be a positive number)
+	if cfg.Modifiers.FirstSeenIncreaseThreshold < 0 {
+		return fmt.Errorf("the first seen modifier increase threshold must be a positive number of days, got %v", cfg.Modifiers.FirstSeenIncreaseThreshold)
+	}
+
+	// validate the configured first seen score decrease modifier values (must be between 0 and 1)
+	if cfg.Modifiers.FirstSeenScoreDecrease < 0 || cfg.Modifiers.FirstSeenScoreDecrease > 1 {
+		return fmt.Errorf("the first seen modifier score decrease must be between 0 and 1, got %v", cfg.Modifiers.FirstSeenScoreDecrease)
+	}
+
+	// validate first seen score decrease threshold (positive number and greater than the increase threshold)
+	if cfg.Modifiers.FirstSeenDecreaseThreshold < 0 {
+		return fmt.Errorf("the first seen modifier decrease threshold must be between 0 and 90 days, got %v", cfg.Modifiers.FirstSeenDecreaseThreshold)
+	}
+	if cfg.Modifiers.FirstSeenDecreaseThreshold <= cfg.Modifiers.FirstSeenIncreaseThreshold {
+		return fmt.Errorf("the first seen modifier decrease threshold must be greater than the increase threshold, got %v", cfg.Modifiers.FirstSeenDecreaseThreshold)
+	}
+
+	// validate the configured missing host count score increase (must be between 0 and 1)
+	if cfg.Modifiers.MissingHostCountScoreIncrease < 0 || cfg.Modifiers.MissingHostCountScoreIncrease > 1 {
+		return fmt.Errorf("the missing host count score increase must be between 0 and 1, got %v", cfg.Modifiers.MissingHostCountScoreIncrease)
+	}
+
+	// validate the configured rare signature score increase
+	if cfg.Modifiers.RareSignatureScoreIncrease < 0 || cfg.Modifiers.RareSignatureScoreIncrease > 1 {
+		return fmt.Errorf("the rare signature score increase must be between 0 and 1, got %v", cfg.Modifiers.RareSignatureScoreIncrease)
+	}
+
+	// validate the configured c2 over DNS direct connection score increase
+	if cfg.Modifiers.C2OverDNSDirectConnScoreIncrease < 0 || cfg.Modifiers.C2OverDNSDirectConnScoreIncrease > 1 {
+		return fmt.Errorf("the c2 over DNS direct connection score increase must be between 0 and 1, got %v", cfg.Modifiers.C2OverDNSDirectConnScoreIncrease)
+	}
+
+	// validate the configured MIME type/URI mismatch score increase
+	if cfg.Modifiers.MIMETypeMismatchScoreIncrease < 0 || cfg.Modifiers.MIMETypeMismatchScoreIncrease > 1 {
+		return fmt.Errorf("the MIME type/URI mismatch score increase must be between 0 and 1, got %v", cfg.Modifiers.MIMETypeMismatchScoreIncrease)
+	}
+
+	// validate log level
+	if cfg.LogLevel < -1 || cfg.LogLevel > 5 {
+		return fmt.Errorf("the LogLevel must be between -1 and 5 (inclusive)")
+	}
+
+	return nil
+}
+
+// validateScoreThresholds validates the score thresholds based on the provided min and max values
+func validateScoreThresholds(s ScoreThresholds, min int, max int) error {
+	// check if values are in increasing order and unique
+	if s.Base >= s.Low || s.Low >= s.Med || s.Med >= s.High {
+		return fmt.Errorf("score thresholds must be in increasing order and unique: %v", s)
+	}
+
+	// validate that base is in range (if min is provided)
+	if min > -1 && s.Base < min {
+		return fmt.Errorf("base score threshold must be greater than or equal to %d", min)
+	}
+
+	// validate that high is in range (if max is provided)
+	if max > -1 && s.High > max {
+		return fmt.Errorf("high score threshold must be less than or equal to %d", max)
+	}
+
+	return nil
+}
+
+// parseImpactCategoryScores sets the corresponding scores for the binary indicators
+func (cfg *Config) parseImpactCategoryScores() error {
+
+	strobeScore, err := GetScoreFromImpactCategory(cfg.Scoring.StrobeImpact.Category)
+	if err != nil {
+		return err
+	}
+
+	cfg.Scoring.StrobeImpact.Score = strobeScore
+
+	threatIntelScore, err := GetScoreFromImpactCategory(cfg.Scoring.ThreatIntelImpact.Category)
+	if err != nil {
+		return err
+	}
+
+	cfg.Scoring.ThreatIntelImpact.Score = threatIntelScore
+
+	return nil
+
+}
+
+// ValidateImpactCategory checks if the provided string is a valid impact value.
+// this function is meant to parse the category from the value a user places in the config
+// Since a score is only critical if its modifiers boost the score over the high category,
+// we do not add the CriticalThreat category here
+func ValidateImpactCategory(value ImpactCategory) error {
+	switch value {
+	case HighThreat, MediumThreat, LowThreat, NoneThreat:
+		return nil
+	default:
+		return errInvalidImpactCategory
+	}
+}
+
+func GetScoreFromImpactCategory(category ImpactCategory) (float32, error) {
+	switch {
+	case category == HighThreat:
+		return HIGH_CATEGORY_SCORE, nil
+	case category == MediumThreat:
+		return MEDIUM_CATEGORY_SCORE, nil
+	case category == LowThreat:
+		return LOW_CATEGORY_SCORE, nil
+	case category == NoneThreat:
+		return NONE_CATEGORY_SCORE, nil
+	}
+	return 0, errInvalidImpactCategory
+}
+
+func GetImpactCategoryFromScore(score float32) ImpactCategory {
+	switch {
+	// >80%
+	case score > MEDIUM_CATEGORY_SCORE:
+		return HighThreat
+		// >40% and <=60%
+	case score > LOW_CATEGORY_SCORE && score <= MEDIUM_CATEGORY_SCORE:
+		return MediumThreat
+		// >20% and <=40%
+	case score > NONE_CATEGORY_SCORE && score <= LOW_CATEGORY_SCORE:
+		return LowThreat
+		// <=20%
+	case score <= NONE_CATEGORY_SCORE:
+		return NoneThreat
+	}
+
+	return NoneThreat
+}
+
+// return a copy of the default config object
+func defaultConfig() Config {
+	return Config{
+		UpdateCheckEnabled: true,
+		Filter: Filter{
+			InternalSubnetsJSON:       []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"},
+			AlwaysIncludedSubnetsJSON: []string{},
+			NeverIncludedSubnetsJSON:  getMandatoryNeverIncludeSubnets(),
+			AlwaysIncludedDomains:     []string{},
+			NeverIncludedDomains:      []string{},
+			FilterExternalToInternal:  true,
+		},
+		HTTPExtensionsFilePath:          "./http_extensions_list.csv",
+		BatchSize:                       100000,
+		MaxQueryExecutionTime:           120,
+		MonthsToKeepHistoricalFirstSeen: 3,
+		Scoring: Scoring{
+			Beacon: Beacon{
+				UniqueConnectionThreshold:       4,
+				TsWeight:                        0.25,
+				DsWeight:                        0.25,
+				DurWeight:                       0.25,
+				HistWeight:                      0.25,
+				DurMinHours:                     6,
+				DurIdealNumberOfConsistentHours: 12,
+				HistModeSensitivity:             0.05,
+				HistBimodalOutlierRemoval:       1,
+				HistBimodalMinHours:             11,
+				ScoreThresholds: ScoreThresholds{
+					Base: 50,
+					Low:  75,
+					Med:  90,
+					High: 100,
+				},
+			},
+
+			LongConnectionMinimumDuration: 1 * 3600, // 1 hour (in seconds)
+			LongConnectionScoreThresholds: ScoreThresholds{
+				Base: 3600,
+				Low:  4 * 3600,
+				Med:  8 * 3600,
+				High: 12 * 3600,
+			},
+
+			C2SubdomainThreshold: 100,
+			C2ScoreThresholds: ScoreThresholds{
+				Base: 100,
+				Low:  500,
+				Med:  800,
+				High: 1000,
+			},
+
+			StrobeImpact: ScoreImpact{Category: HighThreat, Score: HIGH_CATEGORY_SCORE},
+
+			ThreatIntelImpact: ScoreImpact{Category: HighThreat, Score: HIGH_CATEGORY_SCORE},
+		},
+		Modifiers: Modifiers{
+			ThreatIntelScoreIncrease:     0.15,   // score +15% if data size >= 25 MB
+			ThreatIntelDataSizeThreshold: 2.5e+7, // 25 MB (as bytes)
+
+			PrevalenceScoreIncrease:     0.15, // score +15% if prevalence <= 2%
+			PrevalenceIncreaseThreshold: 0.02,
+			PrevalenceScoreDecrease:     0.15, // score -15% if prevalence >= 50%
+			PrevalenceDecreaseThreshold: 0.5,  // must be greater than the increase threshold
+
+			FirstSeenScoreIncrease:     0.15, // score +15% if first seen <= 7 days ago
+			FirstSeenIncreaseThreshold: 7,
+			FirstSeenScoreDecrease:     0.15, // score -15% if first seen >= 30 days ago
+			FirstSeenDecreaseThreshold: 30,   // must be greater than the increase threshold
+			// because the longer a host has been seen on the network, the less sus it is
+
+			MissingHostCountScoreIncrease: 0.10, // +10% score for any (>0) missing hosts
+
+			RareSignatureScoreIncrease: 0.15, // +15% score for connections with a rare signature
+
+			C2OverDNSDirectConnScoreIncrease: 0.15, // +15% score for domains that were queried but had no direct connections
+
+			MIMETypeMismatchScoreIncrease: 0.15, // +15% score for connections with mismatched MIME type/URI
+		},
+		ThreatIntel: ThreatIntel{
+			OnlineFeeds:          []string{},
+			CustomFeedsDirectory: "/etc/rita/threat_intel_feeds",
+		},
+		LogLevel:       1,    // INFO level is default
+		LoggingEnabled: true, // enable logging by default
+	}
+}

config/config_test.go

@@ -0,0 +1,970 @@
+package config
+
+import (
+	"activecm/rita/util"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/joho/godotenv"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// defaultConfigPath specifies the path of RITA's static config file
+const defaultConfigPath = "../config.hjson"
+
+func TestMain(m *testing.M) {
+	// load environment variables with panic prevention
+	if err := godotenv.Overload("../.env", "../integration/test.env"); err != nil {
+		log.Fatalf("Error loading .env file: %v", err)
+	}
+	os.Exit(m.Run())
+}
+
+func TestLoadConfig(t *testing.T) {
+	afs := afero.NewOsFs()
+
+	// load the default config file
+	cfg, err := LoadConfig(afs, defaultConfigPath)
+	require.NoError(t, err, "should be able to load the default config file")
+
+	err = cfg.Validate()
+	require.NoError(t, err, "the loaded default config file should be valid")
+
+	cfg, err = GetConfig()
+	require.NoError(t, err, "should be able to get the config file after it has been loaded")
+
+	err = cfg.Validate()
+	require.NoError(t, err, "the config returned from GetConfig should be valid")
+}
+
+func TestReadFile(t *testing.T) {
+	afs := afero.NewOsFs()
+	fileContents, err := readFile(afs, defaultConfigPath)
+	require.NoError(t, err)
+	assert.NotEmpty(t, fileContents)
+}
+
+func TestParseJSON(t *testing.T) {
+
+	tests := []struct {
+		name           string
+		config         []byte
+		expectedConfig Config
+		expectedError  error
+	}{
+		{
+			name: "valid config",
+			config: []byte(`
+			{
+				db_connection: "localhost:9000",
+				update_check_enabled: false,
+				filtering: {
+					internal_subnets: ["11.0.0.0/8", "120.130.140.150/8"],
+					always_included_subnets: ["13.0.0.0/8", "160.140.150.160/8"],
+					never_included_subnets: ["12.0.0.0/8", "150.140.150.160/8"],
+					always_included_domains: ["abc.com", "def.com"],
+					never_included_domains: ["ghi.com", "jkl.com"],
+					filter_external_to_internal: false,
+				},
+				http_extensions_file_path: "/path/to/http/extensions",
+				batch_size: 75000,
+				max_query_execution_time: 120000,
+				months_to_keep_historical_first_seen: 6,
+				threat_intel: {
+					online_feeds: ["https://example.com/feed1", "https://example.com/feed2"],
+					custom_feeds_directory: "/path/to/custom/feeds",
+				},
+				scoring: {
+					beacon: {
+						unique_connection_threshold: 10,
+						timestamp_score_weight: 0.35,
+						datasize_score_weight: 0.20,
+						duration_score_weight: 0.35,
+						histogram_score_weight: 0.10,
+						duration_min_hours_seen: 10,
+						duration_consistency_ideal_hours_seen: 15,
+						histogram_mode_sensitivity: 0.08,
+						histogram_bimodal_outlier_removal: 2,
+						histogram_bimodal_min_hours_seen: 15,
+						score_thresholds: {
+							base: 0,
+							low: 1,
+							medium: 2,
+							high: 3
+						},
+					},
+					long_connection_minimum_duration: 10,
+					long_connection_score_thresholds: {
+						base: 0,
+						low: 1,
+						medium: 2,
+						high: 3
+					},
+					c2_subdomain_threshold: 10,
+					c2_score_thresholds: {
+						base: 0,
+						low: 1,
+						medium: 2,
+						high: 3
+					},
+					strobe_impact: {
+						category: "low",
+					},
+					threat_intel_impact: {
+						category: "low",
+					},
+				},
+				modifiers: {
+					threat_intel_score_increase: 0.1,
+					threat_intel_datasize_threshold: 100,
+					prevalence_score_increase: 0.6,
+					prevalence_increase_threshold: 0.1,
+					prevalence_score_decrease: 0.1,
+					prevalence_decrease_threshold: 0.2,
+					first_seen_score_increase: 0.8,
+					first_seen_increase_threshold: 10,
+					first_seen_score_decrease: 0.2,
+					first_seen_decrease_threshold: 50,
+					missing_host_count_score_increase: 0.4,
+					rare_signature_score_increase: 0.4,
+					c2_over_dns_direct_conn_score_increase: 0.9,
+					mime_type_mismatch_score_increase: 0.6
+				},
+				log_level: 3,
+				logging_enabled: false,
+			}
+			`),
+			expectedConfig: Config{
+				UpdateCheckEnabled: false,
+				Filter: Filter{
+					InternalSubnetsJSON: []string{"11.0.0.0/8", "120.130.140.150/8"},
+					InternalSubnets: []*net.IPNet{
+						{IP: net.IP{11, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{120, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+					},
+					AlwaysIncludedSubnetsJSON: []string{"13.0.0.0/8", "160.140.150.160/8"},
+					AlwaysIncludedSubnets: []*net.IPNet{
+						{IP: net.IP{13, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{160, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+					},
+					// mandatoryNeverIncludeSubnets are always apended to any neverIncludedSubnet entries
+					NeverIncludedSubnetsJSON: append([]string{"12.0.0.0/8", "150.140.150.160/8"}, getMandatoryNeverIncludeSubnets()...),
+					NeverIncludedSubnets: []*net.IPNet{
+						{IP: net.IP{12, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{150, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{255, 255, 255, 255}},
+						{IP: net.IP{127, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{169, 254, 0, 0}, Mask: net.IPMask{255, 255, 0, 0}},
+						{IP: net.IP{224, 0, 0, 0}, Mask: net.IPMask{240, 0, 0, 0}},
+						{IP: net.IP{255, 255, 255, 255}, Mask: net.IPMask{255, 255, 255, 255}},
+						{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
+						{IP: net.IPv6unspecified, Mask: net.CIDRMask(128, 128)},
+						{IP: net.ParseIP("fe80::"), Mask: net.CIDRMask(10, 128)},
+						{IP: net.ParseIP("ff00::"), Mask: net.CIDRMask(8, 128)},
+						{IP: net.ParseIP("ff02::2"), Mask: net.CIDRMask(128, 128)},
+					},
+
+					AlwaysIncludedDomains:    []string{"abc.com", "def.com"},
+					NeverIncludedDomains:     []string{"ghi.com", "jkl.com"},
+					FilterExternalToInternal: false,
+				},
+				HTTPExtensionsFilePath:          "/path/to/http/extensions",
+				BatchSize:                       75000,
+				MaxQueryExecutionTime:           120000,
+				MonthsToKeepHistoricalFirstSeen: 6,
+				Scoring: Scoring{
+					Beacon: Beacon{
+						UniqueConnectionThreshold:       10,
+						TsWeight:                        0.35,
+						DsWeight:                        0.20,
+						DurWeight:                       0.35,
+						HistWeight:                      0.10,
+						DurMinHours:                     10,
+						DurIdealNumberOfConsistentHours: 15,
+						HistModeSensitivity:             0.08,
+						HistBimodalOutlierRemoval:       2,
+						HistBimodalMinHours:             15,
+						ScoreThresholds: ScoreThresholds{
+							Base: 0,
+							Low:  1,
+							Med:  2,
+							High: 3,
+						},
+					},
+					LongConnectionMinimumDuration: 10,
+					LongConnectionScoreThresholds: ScoreThresholds{
+						Base: 0,
+						Low:  1,
+						Med:  2,
+						High: 3,
+					},
+					C2SubdomainThreshold: 10,
+					C2ScoreThresholds: ScoreThresholds{
+						Base: 0,
+						Low:  1,
+						Med:  2,
+						High: 3,
+					},
+					StrobeImpact: ScoreImpact{
+						Category: LowThreat,
+						Score:    LOW_CATEGORY_SCORE,
+					},
+					ThreatIntelImpact: ScoreImpact{
+						Category: LowThreat,
+						Score:    LOW_CATEGORY_SCORE,
+					},
+				},
+				Modifiers: Modifiers{
+					ThreatIntelScoreIncrease:         0.1,
+					ThreatIntelDataSizeThreshold:     100,
+					PrevalenceScoreIncrease:          0.6,
+					PrevalenceIncreaseThreshold:      0.1,
+					PrevalenceScoreDecrease:          0.1,
+					PrevalenceDecreaseThreshold:      0.2,
+					FirstSeenScoreIncrease:           0.8,
+					FirstSeenIncreaseThreshold:       10,
+					FirstSeenScoreDecrease:           0.2,
+					FirstSeenDecreaseThreshold:       50,
+					MissingHostCountScoreIncrease:    0.4,
+					RareSignatureScoreIncrease:       0.4,
+					C2OverDNSDirectConnScoreIncrease: 0.9,
+					MIMETypeMismatchScoreIncrease:    0.6,
+				},
+				ThreatIntel: ThreatIntel{
+					OnlineFeeds:          []string{"https://example.com/feed1", "https://example.com/feed2"},
+					CustomFeedsDirectory: "/path/to/custom/feeds",
+				},
+				LogLevel:       3,
+				LoggingEnabled: false,
+			},
+			expectedError: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+
+			// set up default config
+			cfg, err := getDefaultConfig()
+			require.NoError(err, "loading default config should not produce an error")
+
+			// parse JSON
+			err = cfg.parseJSON(test.config)
+
+			// if err == nil && !test.expectedError {
+			// 	expectedVal := reflect.ValueOf(test.expectedConfig)
+			// 	actualVal := reflect.ValueOf(cfg)
+			// 	for i := 0; i < expectedVal.NumField(); i++ {
+			// 		expectedField := expectedVal.Field(i)
+			// 		actualField := actualVal.Field(i)
+			// 		fmt.Println("expectedField: ", expectedField.Interface(), "actualField: ", actualField.Interface())
+			// if !reflect.DeepEqual(expectedField.Interface(), actualField.Interface()) {
+			// 	t.Errorf("Field mismatch in %s: expected %v, got %v",
+			// 		expectedVal.Type().Field(i).Name,
+			// 		expectedField.Interface(),
+			// 		actualField.Interface())
+			// }
+			// 	}
+			// }
+
+			// if test.expectedError {
+			// 	require.Error(err, "parseJSON should have produced an error")
+			// } else {
+			// 	require.NoError(err, "parseJSON should not produce an error")
+			// // Compare all fields using reflection
+			// if !reflect.DeepEqual(cfg, test.expectedConfig) {
+			// 	t.Errorf("Config fields do not match. Got %+v, expected %+v", cfg, test.expectedConfig)
+			// }
+			// elemCfg := reflect.ValueOf(cfg)
+			// elemExpected := reflect.ValueOf(test.expectedConfig)
+			// for i := 0; i < elemCfg.NumField(); i++ {
+			// 	cfgField := elemCfg.Field(i)
+			// 	expectedField := elemExpected.Field(i)
+
+			// 	if !reflect.DeepEqual(cfgField.Interface(), expectedField.Interface()) {
+			// 		t.Errorf("Field '%s' mismatch: got %+v, want %+v", elemCfg.Type().Field(i).Name, cfgField.Interface(), expectedField.Interface())
+			// 	}
+			// }
+			// }
+
+			// check if an error was expected
+			require.Equal(test.expectedError, err, "error should match expected value")
+
+			// verify that env variables are not overwritten by JSON
+			// load environment variables with panic prevention
+			err = godotenv.Overload("../.env", "../integration/test.env")
+			require.NoError(err, "loading environment variables should not produce an error")
+			// get the database connection string
+			connection := os.Getenv("DB_ADDRESS")
+			require.Equal(connection, cfg.DBConnection, "DBConnection should not be overwritten by JSON and should match the env variable")
+
+			// check for proper parsing
+			require.Equal(test.expectedConfig.UpdateCheckEnabled, cfg.UpdateCheckEnabled, "UpdateCheckEnabled should match expected value")
+
+			require.ElementsMatch(test.expectedConfig.Filter.InternalSubnetsJSON, cfg.Filter.InternalSubnetsJSON, "InternalSubnetsJSON should match expected value")
+			require.ElementsMatch(test.expectedConfig.Filter.InternalSubnets, cfg.Filter.InternalSubnets, "InternalSubnets should match expected value")
+
+			require.ElementsMatch(test.expectedConfig.Filter.AlwaysIncludedSubnetsJSON, cfg.Filter.AlwaysIncludedSubnetsJSON, "AlwaysIncludedSubnetsJSON should match expected value")
+			require.ElementsMatch(test.expectedConfig.Filter.AlwaysIncludedSubnets, cfg.Filter.AlwaysIncludedSubnets, "AlwaysIncludedSubnets should match expected value")
+
+			require.ElementsMatch(test.expectedConfig.Filter.NeverIncludedSubnetsJSON, cfg.Filter.NeverIncludedSubnetsJSON, "NeverIncludedSubnetsJSON should match expected value")
+			require.ElementsMatch(test.expectedConfig.Filter.NeverIncludedSubnets, cfg.Filter.NeverIncludedSubnets, "NeverIncludedSubnets should match expected value")
+
+			require.ElementsMatch(test.expectedConfig.Filter.AlwaysIncludedDomains, cfg.Filter.AlwaysIncludedDomains, "AlwaysIncludedDomains should match expected value")
+			require.ElementsMatch(test.expectedConfig.Filter.NeverIncludedDomains, cfg.Filter.NeverIncludedDomains, "NeverIncludedDomains should match expected value")
+
+			require.Equal(test.expectedConfig.Filter.FilterExternalToInternal, cfg.Filter.FilterExternalToInternal, "FilterExternalToInternal should match expected value")
+
+			require.Equal(test.expectedConfig.HTTPExtensionsFilePath, cfg.HTTPExtensionsFilePath, "HTTPExtensionsFilePath should match expected value")
+
+			require.Equal(test.expectedConfig.BatchSize, cfg.BatchSize, "BatchSize should match expected value")
+			require.Equal(test.expectedConfig.MaxQueryExecutionTime, cfg.MaxQueryExecutionTime, "MaxQuertExecutionTime should match expected value")
+
+			require.Equal(test.expectedConfig.MonthsToKeepHistoricalFirstSeen, cfg.MonthsToKeepHistoricalFirstSeen, "MonthsToKeepHistoricalFirstSeen should match expected value")
+
+			require.Equal(test.expectedConfig.ThreatIntel.OnlineFeeds, cfg.ThreatIntel.OnlineFeeds, "OnlineFeeds should match expected value")
+			require.Equal(test.expectedConfig.ThreatIntel.CustomFeedsDirectory, cfg.ThreatIntel.CustomFeedsDirectory, "CustomFeedsDirectory should match expected value")
+
+			require.Equal(test.expectedConfig.Scoring.Beacon.UniqueConnectionThreshold, cfg.Scoring.Beacon.UniqueConnectionThreshold, "BeaconUniqueConnectionThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.TsWeight, cfg.Scoring.Beacon.TsWeight, 0.00001, "BeaconTsWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.DsWeight, cfg.Scoring.Beacon.DsWeight, 0.00001, "BeaconDsWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.DurWeight, cfg.Scoring.Beacon.DurWeight, 0.00001, "BeaconDurWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.HistWeight, cfg.Scoring.Beacon.HistWeight, 0.00001, "BeaconHistWeight should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.DurMinHours, cfg.Scoring.Beacon.DurMinHours, "BeaconDurMinHoursSeen should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.DurIdealNumberOfConsistentHours, cfg.Scoring.Beacon.DurIdealNumberOfConsistentHours, "BeaconDurConsistencyIdealHoursSeen should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.HistModeSensitivity, cfg.Scoring.Beacon.HistModeSensitivity, 0.00001, "BeaconHistModeSensitivity should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.HistBimodalOutlierRemoval, cfg.Scoring.Beacon.HistBimodalOutlierRemoval, "BeaconHistBimodalOutlierRemoval should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.HistBimodalMinHours, cfg.Scoring.Beacon.HistBimodalMinHours, "BeaconHistBimodalMinHoursSeen should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Base, cfg.Scoring.Beacon.ScoreThresholds.Base, "BeaconScoreThresholds.Base should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Low, cfg.Scoring.Beacon.ScoreThresholds.Low, "BeaconScoreThresholds.Low should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Med, cfg.Scoring.Beacon.ScoreThresholds.Med, "BeaconScoreThresholds.Med should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.High, cfg.Scoring.Beacon.ScoreThresholds.High, "BeaconScoreThresholds.High should match expected value")
+
+			require.Equal(test.expectedConfig.Scoring.LongConnectionMinimumDuration, cfg.Scoring.LongConnectionMinimumDuration, "LongConnectionMinimumDuration should match expected value")
+			require.Equal(test.expectedConfig.Scoring.LongConnectionScoreThresholds.Base, cfg.Scoring.LongConnectionScoreThresholds.Base, "LongConnectionScoreThresholds.Base should match expected value")
+			require.Equal(test.expectedConfig.Scoring.LongConnectionScoreThresholds.Low, cfg.Scoring.LongConnectionScoreThresholds.Low, "LongConnectionScoreThresholds.Low should match expected value")
+			require.Equal(test.expectedConfig.Scoring.LongConnectionScoreThresholds.Med, cfg.Scoring.LongConnectionScoreThresholds.Med, "LongConnectionScoreThresholds.Med should match expected value")
+			require.Equal(test.expectedConfig.Scoring.LongConnectionScoreThresholds.High, cfg.Scoring.LongConnectionScoreThresholds.High, "LongConnectionScoreThresholds.High should match expected value")
+
+			require.Equal(test.expectedConfig.Scoring.C2SubdomainThreshold, cfg.Scoring.C2SubdomainThreshold, "C2SubdomainThreshold should match expected value")
+			require.Equal(test.expectedConfig.Scoring.C2ScoreThresholds.Base, cfg.Scoring.C2ScoreThresholds.Base, "C2ScoreThresholds.Base should match expected value")
+			require.Equal(test.expectedConfig.Scoring.C2ScoreThresholds.Low, cfg.Scoring.C2ScoreThresholds.Low, "C2ScoreThresholds.Low should match expected value")
+			require.Equal(test.expectedConfig.Scoring.C2ScoreThresholds.Med, cfg.Scoring.C2ScoreThresholds.Med, "C2ScoreThresholds.Med should match expected value")
+			require.Equal(test.expectedConfig.Scoring.C2ScoreThresholds.High, cfg.Scoring.C2ScoreThresholds.High, "C2ScoreThresholds.High should match expected value")
+
+			require.Equal(test.expectedConfig.Scoring.StrobeImpact.Category, cfg.Scoring.StrobeImpact.Category, "StrobeImpact.Category should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.StrobeImpact.Score, cfg.Scoring.StrobeImpact.Score, 0.00001, "StrobeImpact.Score should match expected value")
+
+			require.Equal(test.expectedConfig.Scoring.ThreatIntelImpact.Category, cfg.Scoring.ThreatIntelImpact.Category, "ThreatIntelImpact.Category should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.ThreatIntelImpact.Score, cfg.Scoring.ThreatIntelImpact.Score, 0.00001, "ThreatIntelImpact.Score to be %v, got %v", test.expectedConfig.Scoring.ThreatIntelImpact.Score, cfg.Scoring.ThreatIntelImpact.Score)
+
+			require.InDelta(test.expectedConfig.Modifiers.ThreatIntelScoreIncrease, cfg.Modifiers.ThreatIntelScoreIncrease, 0.00001, "ThreatIntelScoreIncrease should match expected value")
+			require.Equal(test.expectedConfig.Modifiers.ThreatIntelDataSizeThreshold, cfg.Modifiers.ThreatIntelDataSizeThreshold, "ThreatIntelDataSizeThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.PrevalenceScoreIncrease, cfg.Modifiers.PrevalenceScoreIncrease, 0.00001, "PrevalenceScoreIncrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.PrevalenceIncreaseThreshold, cfg.Modifiers.PrevalenceIncreaseThreshold, 0.00001, "PrevalenceIncreaseThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.PrevalenceScoreDecrease, cfg.Modifiers.PrevalenceScoreDecrease, 0.00001, "PrevalenceScoreDecrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.PrevalenceDecreaseThreshold, cfg.Modifiers.PrevalenceDecreaseThreshold, 0.00001, "PrevalenceDecreaseThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.FirstSeenScoreIncrease, cfg.Modifiers.FirstSeenScoreIncrease, 0.00001, "FirstSeenScoreIncrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.FirstSeenIncreaseThreshold, cfg.Modifiers.FirstSeenIncreaseThreshold, 0.00001, "FirstSeenIncreaseThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.FirstSeenScoreDecrease, cfg.Modifiers.FirstSeenScoreDecrease, 0.00001, "FirstSeenScoreDecrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.FirstSeenDecreaseThreshold, cfg.Modifiers.FirstSeenDecreaseThreshold, 0.00001, "FirstSeenDecreaseThreshold should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.MissingHostCountScoreIncrease, cfg.Modifiers.MissingHostCountScoreIncrease, 0.00001, "MissingHostCountScoreIncrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.RareSignatureScoreIncrease, cfg.Modifiers.RareSignatureScoreIncrease, 0.00001, "RareSignatureScoreIncrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.C2OverDNSDirectConnScoreIncrease, cfg.Modifiers.C2OverDNSDirectConnScoreIncrease, 0.00001, "C2OverDNSDirectConnScoreIncrease should match expected value")
+			require.InDelta(test.expectedConfig.Modifiers.MIMETypeMismatchScoreIncrease, cfg.Modifiers.MIMETypeMismatchScoreIncrease, 0.00001, "MIMETypeMismatchScoreIncrease should match expected value")
+
+			require.Equal(test.expectedConfig.LogLevel, cfg.LogLevel, "LogLevel should match expected value")
+			require.Equal(test.expectedConfig.LoggingEnabled, cfg.LoggingEnabled, "LoggingEnabled should match expected value")
+		})
+	}
+}
+
+func TestReadFileConfig(t *testing.T) {
+
+	tests := []struct {
+		name           string
+		configJSON     string
+		expectedConfig Config
+		expectedError  bool
+	}{
+		{
+			name: "valid config",
+			// create a JSON string to write to the temporary file
+			configJSON: `{
+					"db_connection": "localhost:9999",
+					"filtering": {
+						"internal_subnets": ["11.0.0.0/8", "120.130.140.150/8"],
+						"always_included_subnets": [],
+						"never_included_subnets": ["::1/128", "12.0.0.0/8", "150.140.150.160/8"],
+						"always_included_domains": [],
+						"never_included_domains": [],
+						"filter_external_to_internal": true,
+					},
+					scoring: {
+						beacon: {
+							"unique_connection_threshold": 10,
+							"timestamp_score_weight": 0.35,
+							"datasize_score_weight": 0.20,
+							"duration_score_weight": 0.35,
+							"histogram_score_weight": 0.10,
+							"score_thresholds": {
+								"base": 0,
+								"low": 1,
+								"medium": 2,
+								"high": 3
+							}
+						}
+					}	
+				}`,
+			expectedConfig: Config{
+				Filter: Filter{
+					InternalSubnetsJSON: []string{"11.0.0.0/8", "120.130.140.150/8"},
+					InternalSubnets: []*net.IPNet{
+						{IP: net.IP{11, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{120, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+					},
+					AlwaysIncludedSubnetsJSON: []string{},
+					AlwaysIncludedSubnets:     nil,
+					// mandatoryNeverIncludeSubnets are always apended to any neverIncludedSubnet entries
+					// in this case we are including one of the mandatoryNeverIncludeSubnets in the neverIncludedSubnets list
+					// to test that the mandatory entries are not duplicated when they are appended
+					NeverIncludedSubnetsJSON: util.EnsureSliceContainsAll([]string{"::1/128", "12.0.0.0/8", "150.140.150.160/8"}, getMandatoryNeverIncludeSubnets()),
+					NeverIncludedSubnets: []*net.IPNet{
+						{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}, // would normally be appended with mandatory values at the end of config entries
+						{IP: net.IP{12, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{150, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{0, 0, 0, 0}, Mask: net.IPMask{255, 255, 255, 255}},
+						{IP: net.IP{127, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+						{IP: net.IP{169, 254, 0, 0}, Mask: net.IPMask{255, 255, 0, 0}},
+						{IP: net.IP{224, 0, 0, 0}, Mask: net.IPMask{240, 0, 0, 0}},
+						{IP: net.IP{255, 255, 255, 255}, Mask: net.IPMask{255, 255, 255, 255}},
+						{IP: net.IPv6unspecified, Mask: net.CIDRMask(128, 128)},
+						{IP: net.ParseIP("fe80::"), Mask: net.CIDRMask(10, 128)},
+						{IP: net.ParseIP("ff00::"), Mask: net.CIDRMask(8, 128)},
+						{IP: net.ParseIP("ff02::2"), Mask: net.CIDRMask(128, 128)},
+					},
+
+					AlwaysIncludedDomains:    []string{},
+					NeverIncludedDomains:     []string{},
+					FilterExternalToInternal: true,
+				},
+				Scoring: Scoring{
+					Beacon: Beacon{
+						UniqueConnectionThreshold: 10,
+						TsWeight:                  0.35,
+						DsWeight:                  0.20,
+						DurWeight:                 0.35,
+						HistWeight:                0.10,
+						ScoreThresholds: ScoreThresholds{
+							Base: 0,
+							Low:  1,
+							Med:  2,
+							High: 3,
+						},
+					},
+				},
+			},
+			expectedError: false,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+
+			// create mock file system in memory
+			afs := afero.NewMemMapFs()
+
+			// get config file path
+			configPath := fmt.Sprintf("test-config-%d.hjson", i)
+
+			// create a temporary file to read from
+			file, err := afs.Create(configPath)
+			require.NoError(err, "creating file should not produce an error")
+
+			// set file permissions
+			err = afs.Chmod(configPath, os.FileMode(0o775))
+			require.NoError(err, "changing file permissions should not produce an error")
+
+			// write the JSON to temporary file
+			bytesWritten, err := file.Write([]byte(test.configJSON))
+			require.NoError(err, "writing data to file should not produce an error")
+			require.Equal(len(test.configJSON), bytesWritten, "number of bytes written should be equal to the length of the mock data")
+
+			// close temporary file
+			err = file.Close()
+			require.NoError(err, "closing file should not produce an error")
+
+			// call function
+			cfg, err := ReadFileConfig(afs, configPath)
+			require.NoError(err, "Expected no error when reading file config, got err=%v", err)
+
+			// verify that env variables are not overwritten by JSON
+			// load environment variables with panic prevention
+			err = godotenv.Overload("../.env", "../integration/test.env")
+			require.NoError(err, "loading environment variables should not produce an error")
+			// get the database connection string
+			connection := os.Getenv("DB_ADDRESS")
+			fmt.Println("connection: ", connection)
+			require.Equal(connection, cfg.DBConnection, "DBConnection should not be overwritten by JSON and should match the env variable")
+
+			// verify parsed values
+			require.Equal(test.expectedConfig.Filter.InternalSubnetsJSON, cfg.Filter.InternalSubnetsJSON, "Expected InternalSubnetsJSON to be %v, got %v", test.expectedConfig.Filter.InternalSubnetsJSON, cfg.Filter.InternalSubnetsJSON)
+			require.Equal(test.expectedConfig.Filter.InternalSubnets, cfg.Filter.InternalSubnets, "Expected InternalSubnets to be %v, got %v", test.expectedConfig.Filter.InternalSubnets, cfg.Filter.InternalSubnets)
+
+			require.Equal(test.expectedConfig.Filter.AlwaysIncludedSubnetsJSON, cfg.Filter.AlwaysIncludedSubnetsJSON, "Expected AlwaysIncludedSubnetsJSON to be %v, got %v", test.expectedConfig.Filter.AlwaysIncludedSubnetsJSON, cfg.Filter.AlwaysIncludedSubnetsJSON)
+			require.Equal(test.expectedConfig.Filter.AlwaysIncludedSubnets, cfg.Filter.AlwaysIncludedSubnets, "Expected AlwaysIncludedSubnets to be %v, got %v", test.expectedConfig.Filter.AlwaysIncludedSubnets, cfg.Filter.AlwaysIncludedSubnets)
+
+			require.Equal(test.expectedConfig.Filter.NeverIncludedSubnetsJSON, cfg.Filter.NeverIncludedSubnetsJSON, "Expected NeverIncludedSubnetsJSON to be %v, got %v", test.expectedConfig.Filter.NeverIncludedSubnetsJSON, cfg.Filter.NeverIncludedSubnetsJSON)
+			require.ElementsMatch(test.expectedConfig.Filter.NeverIncludedSubnets, cfg.Filter.NeverIncludedSubnets, "Expected NeverIncludedSubnets to be %v, got %v", test.expectedConfig.Filter.NeverIncludedSubnets, cfg.Filter.NeverIncludedSubnets)
+
+			require.Equal(test.expectedConfig.Filter.AlwaysIncludedDomains, cfg.Filter.AlwaysIncludedDomains, "Expected AlwaysIncludedDomains to be %v, got %v", test.expectedConfig.Filter.AlwaysIncludedDomains, cfg.Filter.AlwaysIncludedDomains)
+			require.Equal(test.expectedConfig.Filter.NeverIncludedDomains, cfg.Filter.NeverIncludedDomains, "Expected NeverIncludedDomains to be %v, got %v", test.expectedConfig.Filter.NeverIncludedDomains, cfg.Filter.NeverIncludedDomains)
+
+			require.Equal(test.expectedConfig.Filter.FilterExternalToInternal, cfg.Filter.FilterExternalToInternal, "Expected FilterExternalToInternal to be %v, got %v", test.expectedConfig.Filter.FilterExternalToInternal, cfg.Filter.FilterExternalToInternal)
+
+			require.Equal(test.expectedConfig.Scoring.Beacon.UniqueConnectionThreshold, cfg.Scoring.Beacon.UniqueConnectionThreshold, "Expected BeaconUniqueConnectionThreshold to be %v, got %v", test.expectedConfig.Scoring.Beacon.UniqueConnectionThreshold, cfg.Scoring.Beacon.UniqueConnectionThreshold)
+			require.InDelta(test.expectedConfig.Scoring.Beacon.TsWeight, cfg.Scoring.Beacon.TsWeight, 0.00001, "BeaconTsWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.DsWeight, cfg.Scoring.Beacon.DsWeight, 0.00001, "BeaconDsWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.DurWeight, cfg.Scoring.Beacon.DurWeight, 0.00001, "BeaconDurWeight should match expected value")
+			require.InDelta(test.expectedConfig.Scoring.Beacon.HistWeight, cfg.Scoring.Beacon.HistWeight, 0.00001, "BeaconHistWeight should match expected value")
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Base, cfg.Scoring.Beacon.ScoreThresholds.Base, "Expected BeaconScoreThresholds.Base to be %v, got %v", test.expectedConfig.Scoring.Beacon.ScoreThresholds.Base, cfg.Scoring.Beacon.ScoreThresholds.Base)
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Low, cfg.Scoring.Beacon.ScoreThresholds.Low, "Expected BeaconScoreThresholds.Low to be %v, got %v", test.expectedConfig.Scoring.Beacon.ScoreThresholds.Low, cfg.Scoring.Beacon.ScoreThresholds.Low)
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.Med, cfg.Scoring.Beacon.ScoreThresholds.Med, "Expected BeaconScoreThresholds.Med to be %v, got %v", test.expectedConfig.Scoring.Beacon.ScoreThresholds.Med, cfg.Scoring.Beacon.ScoreThresholds.Med)
+			require.Equal(test.expectedConfig.Scoring.Beacon.ScoreThresholds.High, cfg.Scoring.Beacon.ScoreThresholds.High, "Expected BeaconScoreThresholds.High to be %v, got %v", test.expectedConfig.Scoring.Beacon.ScoreThresholds.High, cfg.Scoring.Beacon.ScoreThresholds.High)
+
+			// clean up after the test
+			err = afs.Remove(configPath)
+			require.NoError(err, "removing temporary file should not produce an error")
+		})
+	}
+
+}
+
+func TestVerifyBeaconConfig(t *testing.T) {
+	require := require.New(t)
+	// get default config
+	cfg, err := getDefaultConfig()
+	require.NoError(err, "getDefaultConfig should not produce an error")
+
+	// verify the default config
+	err = cfg.verifyConfig()
+	require.NoError(err, "verifyConfig should not produce an error")
+	require.Equal(int64(4), cfg.Scoring.Beacon.UniqueConnectionThreshold, "BeaconUniqueConnectionThreshold should match expected value")
+	require.InDelta(0.25, cfg.Scoring.Beacon.TsWeight, 0.00001, "BeaconTsWeight should match expected value")
+	require.InDelta(0.25, cfg.Scoring.Beacon.DsWeight, 0.00001, "BeaconDsWeight should match expected value")
+	require.InDelta(0.25, cfg.Scoring.Beacon.DurWeight, 0.00001, "BeaconDurWeight should match expected value")
+	require.InDelta(0.25, cfg.Scoring.Beacon.HistWeight, 0.00001, "BeaconHistWeight should match expected value")
+	require.Equal(6, cfg.Scoring.Beacon.DurMinHours, "BeaconDurMinHoursSeen should match expected value")
+	require.Equal(12, cfg.Scoring.Beacon.DurIdealNumberOfConsistentHours, "BeaconDurIdealNumberOfConsistentHoursSeen should match expected value")
+	require.InDelta(0.05, cfg.Scoring.Beacon.HistModeSensitivity, 0.00001, "BeaconHistModeSensitivity should match expected value")
+	require.Equal(1, cfg.Scoring.Beacon.HistBimodalOutlierRemoval, "BeaconHistBimodalOutlierRemoval should match expected value")
+	require.Equal(11, cfg.Scoring.Beacon.HistBimodalMinHours, "BeaconHistBimodalMinHoursSeen should match expected value")
+}
+
+func TestResetConfig(t *testing.T) {
+	require := require.New(t)
+
+	// get default config
+	origConfig, err := getDefaultConfig()
+	require.NoError(err, "Expected no error when getting default config, got err=%v", err)
+
+	// create a copy of the config
+	// cfg, err := getDefaultConfig()
+	// require.NoError(err, "Expected no error when getting default config, got err=%v", err)
+	cfg := origConfig
+
+	// set some invalid values
+	cfg.Scoring.Beacon.UniqueConnectionThreshold = 1
+	cfg.Scoring.Beacon.TsWeight = 0.5
+	cfg.Scoring.Beacon.DsWeight = 0.5
+	cfg.Scoring.Beacon.DurWeight = 0.5
+	cfg.Scoring.Beacon.HistWeight = 0.5
+	cfg.Scoring.Beacon.DurMinHours = 0
+	cfg.Scoring.Beacon.DurIdealNumberOfConsistentHours = 0
+	cfg.Scoring.Beacon.HistModeSensitivity = 0
+	cfg.Scoring.Beacon.HistBimodalOutlierRemoval = 0
+	cfg.Scoring.Beacon.HistBimodalMinHours = 0
+	cfg.Scoring.Beacon.ScoreThresholds = ScoreThresholds{
+		Base: -1,
+		Low:  -2,
+		Med:  -3,
+		High: -4,
+	}
+
+	// verify that the values are not the same before resetting
+	require.NotEqual(origConfig, cfg, "config should not match default config")
+
+	// reset the config
+	err = cfg.ResetConfig()
+	require.NoError(err, "resetting config should not produce an error")
+
+	// verify that the values have been reset
+	require.Equal(origConfig, cfg, "config should match expected value")
+
+	// verify the config
+	err = cfg.verifyConfig()
+	require.NoError(err, "verifyConfig should not produce an error")
+}
+
+func TestGetDefaultConfig(t *testing.T) {
+	require := require.New(t)
+	cfg, err := getDefaultConfig()
+	require.NoError(err, "getDefaultConfig should not produce an error")
+
+	// get default config variable
+	origConfigVar := defaultConfig()
+
+	// get the database connection string
+	connection := os.Getenv("DB_ADDRESS")
+	require.NotEmpty(connection, "DB_ADDRESS should not be empty")
+	origConfigVar.DBConnection = connection
+
+	// parse the filter variables from the default config variable by hand to ensure they are correctly
+
+	// parse internal subnets
+	internalSubnetList, err := util.ParseSubnets(origConfigVar.Filter.InternalSubnetsJSON)
+	require.NoError(err, "parseSubnets should not produce an error")
+	origConfigVar.Filter.InternalSubnets = internalSubnetList
+
+	// parse never included subnets
+	origConfigVar.Filter.NeverIncludedSubnetsJSON = getMandatoryNeverIncludeSubnets()
+	neverIncludedSubnetList, err := util.ParseSubnets(origConfigVar.Filter.NeverIncludedSubnetsJSON)
+	require.NoError(err, "parseSubnets should not produce an error")
+	origConfigVar.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+
+	// parse always included subnets
+	alwayIncludedSubnetList, err := util.ParseSubnets(origConfigVar.Filter.AlwaysIncludedSubnetsJSON)
+	require.NoError(err, "parseSubnets should not produce an error")
+	origConfigVar.Filter.AlwaysIncludedSubnets = alwayIncludedSubnetList
+
+	// verify that the object returned by the getDefaultConfig function is correct
+	require.Equal(origConfigVar.DBConnection, cfg.DBConnection, "config db connection should match expected value")
+	require.Equal(origConfigVar.UpdateCheckEnabled, cfg.UpdateCheckEnabled, "config update check enabled should match expected value")
+	require.Equal(origConfigVar.Filter, cfg.Filter, "config internal subnets should match expected value")
+	require.Equal(origConfigVar.HTTPExtensionsFilePath, cfg.HTTPExtensionsFilePath, "config http extensions file path should match expected value")
+	require.Equal(origConfigVar.BatchSize, cfg.BatchSize, "config batch size should match expected value")
+	require.Equal(origConfigVar.MonthsToKeepHistoricalFirstSeen, cfg.MonthsToKeepHistoricalFirstSeen, "config months to keep historical first seen should match expected value")
+	require.Equal(origConfigVar.Scoring, cfg.Scoring, "config scoring should match expected value")
+	require.Equal(origConfigVar.Modifiers, cfg.Modifiers, "config modifiers should match expected value")
+	require.Equal(origConfigVar.ThreatIntel, cfg.ThreatIntel, "config threat intel should match expected value")
+	require.Equal(origConfigVar.LogLevel, cfg.LogLevel, "config log level should match expected value")
+	require.Equal(origConfigVar.LoggingEnabled, cfg.LoggingEnabled, "config logging enabled should match expected value")
+
+	// match the whole object just in case
+	require.Equal(origConfigVar, cfg, "config should match expected value")
+}
+
+func TestValidateScoreThresholds(t *testing.T) {
+	tests := []struct {
+		name          string
+		thresholds    ScoreThresholds
+		min           int
+		max           int
+		expectedError bool
+	}{
+		{
+			name: "valid thresholds, (0 - 10)",
+			thresholds: ScoreThresholds{
+				Base: 0,
+				Low:  1,
+				Med:  2,
+				High: 3,
+			},
+			min:           0,
+			max:           10,
+			expectedError: false,
+		},
+		{
+			name: "valid beacon thresholds, (0 - 100)",
+			thresholds: ScoreThresholds{
+				Base: 50,
+				Low:  75,
+				Med:  90,
+				High: 100,
+			},
+			min:           0,
+			max:           100,
+			expectedError: false,
+		},
+		{
+			name: "valid long conn thresholds, (0 - 24*3600)",
+			thresholds: ScoreThresholds{
+				Base: 3600,
+				Low:  4 * 3600,
+				Med:  8 * 3600,
+				High: 12 * 3600,
+			},
+			min:           0,
+			max:           24 * 3600,
+			expectedError: false,
+		},
+		{
+			name: "valid c2 thresholds, (0 - no max)",
+			thresholds: ScoreThresholds{
+				Base: 100,
+				Low:  500,
+				Med:  800,
+				High: 1000,
+			},
+			min:           0,
+			max:           -1,
+			expectedError: false,
+		},
+		{
+			name: "invalid thresholds (not in ascending order)",
+			thresholds: ScoreThresholds{
+				Base: 0,
+				Low:  1,
+				Med:  2,
+				High: 1,
+			},
+			min:           0,
+			max:           10,
+			expectedError: true,
+		},
+		{
+			name: "invalid thresholds (out of range - max)",
+			thresholds: ScoreThresholds{
+				Base: 0,
+				Low:  1,
+				Med:  2,
+				High: 3,
+			},
+			min:           0,
+			max:           2,
+			expectedError: true,
+		},
+		{
+			name: "invalid thresholds (out of range - min)",
+			thresholds: ScoreThresholds{
+				Base: 0,
+				Low:  1,
+				Med:  2,
+				High: 3,
+			},
+			min:           1,
+			max:           10,
+			expectedError: true,
+		},
+		{
+			name: "invalid thresholds (two thresholds equal)",
+			thresholds: ScoreThresholds{
+				Base: 0,
+				Low:  1,
+				Med:  1,
+				High: 3,
+			},
+			min:           0,
+			max:           10,
+			expectedError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+			err := validateScoreThresholds(test.thresholds, test.min, test.max)
+			require.Equal(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err)
+		})
+	}
+}
+
+func TestParseImpactCategoryScores(t *testing.T) {
+	t.Run("Valid Categories", func(t *testing.T) {
+		cfg := &Config{
+			Scoring: Scoring{
+				StrobeImpact: ScoreImpact{
+					Category: HighThreat,
+				},
+				ThreatIntelImpact: ScoreImpact{
+					Category: LowThreat,
+				},
+			},
+		}
+
+		err := cfg.parseImpactCategoryScores()
+		require.NoError(t, err)
+		require.InDelta(t, float32(HIGH_CATEGORY_SCORE), cfg.Scoring.StrobeImpact.Score, 0.0001, "StrobeImpact.Score should match expected value")
+		require.InDelta(t, float32(LOW_CATEGORY_SCORE), cfg.Scoring.ThreatIntelImpact.Score, 0.0001, "ThreatIntelImpact.Score should match expected value")
+	})
+
+	t.Run("More Valid Categories", func(t *testing.T) {
+		cfg := &Config{
+			Scoring: Scoring{
+				StrobeImpact: ScoreImpact{
+					Category: MediumThreat,
+				},
+				ThreatIntelImpact: ScoreImpact{
+					Category: NoneThreat,
+				},
+			},
+		}
+
+		err := cfg.parseImpactCategoryScores()
+		require.NoError(t, err)
+		require.InDelta(t, float32(MEDIUM_CATEGORY_SCORE), cfg.Scoring.StrobeImpact.Score, 0.0001, "StrobeImpact.Score should match expected value")
+		require.InDelta(t, float32(NONE_CATEGORY_SCORE), cfg.Scoring.ThreatIntelImpact.Score, 0.0001, "ThreatIntelImpact.Score should match expected value")
+	})
+
+	t.Run("Invalid Category for StrobeImpact", func(t *testing.T) {
+		cfg := &Config{
+			Scoring: Scoring{
+				StrobeImpact: ScoreImpact{
+					Category: "unknown",
+				},
+				ThreatIntelImpact: ScoreImpact{
+					Category: LowThreat,
+				},
+			},
+		}
+
+		err := cfg.parseImpactCategoryScores()
+		require.Error(t, err)
+	})
+
+	t.Run("Invalid Category for ThreatIntelImpact", func(t *testing.T) {
+		cfg := &Config{
+			Scoring: Scoring{
+				StrobeImpact: ScoreImpact{
+					Category: HighThreat,
+				},
+				ThreatIntelImpact: ScoreImpact{
+					Category: "invalid",
+				},
+			},
+		}
+
+		err := cfg.parseImpactCategoryScores()
+		require.Error(t, err)
+	})
+}
+
+func TestValidateImpactCategory(t *testing.T) {
+	tests := []struct {
+		name          string
+		impact        ImpactCategory
+		expectedError bool
+	}{
+		{
+			name:          "high impact category",
+			impact:        "high",
+			expectedError: false,
+		},
+		{
+			name:          "none impact category",
+			impact:        "none",
+			expectedError: false,
+		},
+		{
+			name:          "invalid impact category",
+			impact:        "iaminvalid",
+			expectedError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+			err := ValidateImpactCategory(test.impact)
+			require.Equal(test.expectedError, err != nil, "Expected error to be %v, got %v", test.expectedError, err)
+		})
+	}
+}
+
+func TestGetScoreFromImpactCategory(t *testing.T) {
+	tests := []struct {
+		name          string
+		impact        ImpactCategory
+		expectedScore float32
+		expectedError error
+	}{
+		{
+			name:          "high impact category",
+			impact:        "high",
+			expectedScore: HIGH_CATEGORY_SCORE,
+			expectedError: nil,
+		},
+		{
+			name:          "medium impact category",
+			impact:        "medium",
+			expectedScore: MEDIUM_CATEGORY_SCORE,
+			expectedError: nil,
+		},
+		{
+			name:          "low impact category",
+			impact:        "low",
+			expectedScore: LOW_CATEGORY_SCORE,
+			expectedError: nil,
+		},
+		{
+			name:          "none impact category",
+			impact:        "none",
+			expectedScore: NONE_CATEGORY_SCORE,
+			expectedError: nil,
+		},
+		{
+			name:          "invalid impact category",
+			impact:        "iaminvalid",
+			expectedScore: 0,
+			expectedError: errInvalidImpactCategory,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+			score, err := GetScoreFromImpactCategory(test.impact)
+			require.Equal(test.expectedError, err, "error should match expected value")
+			require.InDelta(test.expectedScore, score, 0.0001, "score should match expected value")
+		})
+	}
+}
+
+func TestGetImpactCategoryFromScore(t *testing.T) {
+	tests := []struct {
+		name           string
+		score          float32
+		expectedImpact ImpactCategory
+		expectedError  bool
+	}{
+		{
+			name: "high impact category",
+			// score > MEDIUM_CATEGORY_SCORE
+			score:          HIGH_CATEGORY_SCORE,
+			expectedImpact: "high",
+		},
+		{
+			name: "medium impact category",
+			// score > LOW_CATEGORY_SCORE && score <= MEDIUM_CATEGORY_SCORE
+			score:          MEDIUM_CATEGORY_SCORE,
+			expectedImpact: "medium",
+		},
+		{
+			name: "low impact category",
+			// score > NONE_CATEGORY_SCORE && score <= LOW_CATEGORY_SCORE
+			score:          LOW_CATEGORY_SCORE,
+			expectedImpact: "low",
+		},
+		{
+			name: "none impact category",
+			// score <= NONE_CATEGORY_SCORE
+			score:          NONE_CATEGORY_SCORE,
+			expectedImpact: "none",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require := require.New(t)
+			impact := GetImpactCategoryFromScore(test.score)
+			require.Equal(test.expectedImpact, impact, "Expected impact to be %v, got %v", test.expectedImpact, impact)
+		})
+	}
+}

config/filter.go

@@ -0,0 +1,277 @@
+package config
+
+import (
+	"activecm/rita/util"
+
+	"net"
+)
+
+// Filter provides methods for excluding IP addresses, domains, and determining proxy servers during the import step
+// based on the user configuration
+type Filter struct {
+	InternalSubnetsJSON []string `json:"internal_subnets"`
+	InternalSubnets     []*net.IPNet
+
+	AlwaysIncludedSubnetsJSON []string `json:"always_included_subnets"`
+	AlwaysIncludedSubnets     []*net.IPNet
+
+	NeverIncludedSubnetsJSON []string `json:"never_included_subnets"`
+	NeverIncludedSubnets     []*net.IPNet
+
+	AlwaysIncludedDomains []string `json:"always_included_domains"`
+	NeverIncludedDomains  []string `json:"never_included_domains"`
+
+	FilterExternalToInternal bool `json:"filter_external_to_internal"`
+}
+
+func getMandatoryNeverIncludeSubnets() []string {
+	// s2 := make([]string, len(mandatoryNeverIncludeSubnets))
+
+	// _ = copy(s2, mandatoryNeverIncludeSubnets) // s2 is now an independent copy of s
+	// return s2
+	return []string{
+		"0.0.0.0/32",         // current host
+		"127.0.0.0/8",        // loopback
+		"169.254.0.0/16",     // link local
+		"224.0.0.0/4",        // multicast
+		"255.255.255.255/32", // limited broadcast
+		"::1/128",            // loopback
+		"::",                 // unspecified IPv6
+		"fe80::/10",          // link local
+		"ff00::/8",           // multicast
+		"ff02::2",            // local multicast
+	}
+}
+
+func (cfg *Config) parseFilter() error {
+	// parse internal subnets
+	internalSubnetList, err := util.ParseSubnets(cfg.Filter.InternalSubnetsJSON)
+	if err != nil {
+		return err
+	}
+	cfg.Filter.InternalSubnets = internalSubnetList
+
+	// parse always included subnets
+	alwaysIncludedSubnetList, err := util.ParseSubnets(cfg.Filter.AlwaysIncludedSubnetsJSON)
+	if err != nil {
+		return err
+	}
+	cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+
+	// validate that all mandatory never include subnets are present
+	cfg.Filter.NeverIncludedSubnetsJSON = util.EnsureSliceContainsAll(cfg.Filter.NeverIncludedSubnetsJSON, getMandatoryNeverIncludeSubnets())
+
+	// parse never included subnets
+	neverIncludedSubnetList, err := util.ParseSubnets(cfg.Filter.NeverIncludedSubnetsJSON)
+	if err != nil {
+		return err
+	}
+	cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+
+	return nil
+}
+
+// FilterSNIPair returns true if a SNI connection pair is filtered/excluded.
+func (fs *Filter) FilterSNIPair(srcIP net.IP) bool {
+	// check if src is internal
+	isSrcInternal := util.ContainsIP(fs.InternalSubnets, srcIP)
+
+	// filter out connections that have external source IPs
+	return !isSrcInternal
+}
+
+// FilterConnPairForHTTP returns true if a connection pair is filtered
+// based on criteria that should apply regardless of whether or not there is a proxy connection for it
+func (fs *Filter) FilterConnPairForHTTP(srcIP net.IP, dstIP net.IP) bool {
+
+	// check if on always included list
+	isSrcIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, srcIP)
+	isDstIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, dstIP)
+
+	// check if on never included list
+	isSrcExcluded := util.ContainsIP(fs.NeverIncludedSubnets, srcIP)
+	isDstExcluded := util.ContainsIP(fs.NeverIncludedSubnets, dstIP)
+
+	// if either IP is on the AlwaysInclude list, filter does not apply
+	if isSrcIncluded || isDstIncluded {
+		return false
+	}
+
+	// if either IP is on the NeverInclude list, filter applies
+	if isSrcExcluded || isDstExcluded {
+		return true
+	}
+
+	// check if src and dst are internal
+	isSrcInternal := util.ContainsIP(fs.InternalSubnets, srcIP)
+	isDstInternal := util.ContainsIP(fs.InternalSubnets, dstIP)
+
+	// if both addresses are external, filter applies
+	if (!isSrcInternal) && (!isDstInternal) {
+		return true
+	}
+
+	return false
+}
+
+// filterConnPair returns true if a connection pair is filtered/excluded.
+// This is determined by the following rules, in order:
+//  1. Not filtered if either IP is on the AlwaysInclude list
+//  2. Filtered if either IP is on the NeverInclude list
+//  3. Not filtered if InternalSubnets is empty
+//  4. Filtered if both IPs are internal or both are external
+//  5. Filtered if the source IP is external and the destination IP is internal and FilterExternalToInternal has been set in the configuration file
+//  6. Not filtered in all other cases
+func (fs *Filter) FilterConnPair(srcIP net.IP, dstIP net.IP) bool {
+
+	// check if on always included list
+	isSrcIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, srcIP)
+	isDstIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, dstIP)
+
+	// check if on never included list
+	isSrcExcluded := util.ContainsIP(fs.NeverIncludedSubnets, srcIP)
+	isDstExcluded := util.ContainsIP(fs.NeverIncludedSubnets, dstIP)
+
+	// if either IP is on the AlwaysInclude list, filter does not apply
+	if isSrcIncluded || isDstIncluded {
+		return false
+	}
+
+	// if either IP is on the NeverInclude list, filter applies
+	if isSrcExcluded || isDstExcluded {
+		return true
+	}
+
+	// if no internal subnets are defined, return false
+	// note: this should not happen since we validate the config to ensure
+	// that internal subnets is not empty
+	if len(fs.InternalSubnets) == 0 {
+		return false
+	}
+
+	// check if src and dst are internal
+	isSrcInternal := util.ContainsIP(fs.InternalSubnets, srcIP)
+	isDstInternal := util.ContainsIP(fs.InternalSubnets, dstIP)
+
+	// if both addresses are internal, filter applies
+	if isSrcInternal && isDstInternal {
+		return true
+	}
+
+	// if both addresses are external, filter applies
+	if (!isSrcInternal) && (!isDstInternal) {
+		return true
+	}
+
+	// filter external to internal traffic if the user has specified to do so
+	if fs.FilterExternalToInternal && (!isSrcInternal) && isDstInternal {
+		return true
+	}
+
+	// default to not filter the connection pair
+	return false
+}
+
+// filterDNSPair returns true if a DNS connection pair is filtered/excluded.
+// DNS is treated specially since we need to capture internal -> internal DNS traffic
+// in order to detect C2 over DNS with an internal resolver.
+// This is determined by the following rules, in order:
+//  1. Not filtered if either IP is on the AlwaysInclude list
+//  2. Filtered if either IP is on the NeverInclude list
+//  3. Not filtered if InternalSubnets is empty
+//  4. Filtered if both IPs are external (this is different from filterConnPair which filters internal to internal connections)
+//  5. Filtered if the source IP is external and the destination IP is internal and FilterExternalToInternal has been set in the configuration file
+//  6. Not filtered in all other cases
+func (fs *Filter) FilterDNSPair(srcIP net.IP, dstIP net.IP) bool {
+	// check if on always included list
+	isSrcIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, srcIP)
+	isDstIncluded := util.ContainsIP(fs.AlwaysIncludedSubnets, dstIP)
+
+	// check if on never included list
+	isSrcExcluded := util.ContainsIP(fs.NeverIncludedSubnets, srcIP)
+	isDstExcluded := util.ContainsIP(fs.NeverIncludedSubnets, dstIP)
+
+	// if either IP is on the AlwaysInclude list, filter does not apply
+	if isSrcIncluded || isDstIncluded {
+		return false
+	}
+
+	// if either IP is on the NeverInclude list, filter applies
+	if isSrcExcluded || isDstExcluded {
+		return true
+	}
+
+	// if no internal subnets are defined, filter does not apply
+	// this is was the default behavior before InternalSubnets was added
+	if len(fs.InternalSubnets) == 0 {
+		return false
+	}
+
+	// check if src and dst are internal
+	isSrcInternal := util.ContainsIP(fs.InternalSubnets, srcIP)
+	isDstInternal := util.ContainsIP(fs.InternalSubnets, dstIP)
+
+	// if both addresses are external, filter applies
+	if (!isSrcInternal) && (!isDstInternal) {
+		return true
+	}
+
+	// filter external to internal traffic if the user has specified to do so
+	if fs.FilterExternalToInternal && (!isSrcInternal) && isDstInternal {
+		return true
+	}
+
+	// default to not filter the connection pair
+	return false
+}
+
+// filterSingleIP returns true if an IP is filtered/excluded.
+// This is determined by the following rules, in order:
+//  1. Not filtered IP is on the AlwaysInclude list
+//  2. Filtered IP is on the NeverInclude list
+//  3. Not filtered in all other cases
+func (fs *Filter) FilterSingleIP(ip net.IP) bool {
+
+	// check if on always included list
+	if util.ContainsIP(fs.AlwaysIncludedSubnets, ip) {
+		return false
+	}
+
+	// check if on never included list
+	if util.ContainsIP(fs.NeverIncludedSubnets, ip) {
+		return true
+	}
+
+	// default to not filter the IP address
+	return false
+}
+
+// FilterDomain returns true if a domain is filtered/excluded.
+// This is determined by the following rules, in order:
+//  1. Not filtered if domain is on the AlwaysInclude list
+//  2. Filtered if domain is on the NeverInclude list
+//  3. Not filtered in all other cases
+func (fs *Filter) FilterDomain(domain string) bool {
+	// check if on always included list
+	isDomainIncluded := util.ContainsDomain(fs.AlwaysIncludedDomains, domain)
+
+	// check if on never included list
+	isDomainExcluded := util.ContainsDomain(fs.NeverIncludedDomains, domain)
+
+	// if either IP is on the AlwaysInclude list, filter does not apply
+	if isDomainIncluded {
+		return false
+	}
+
+	// if either IP is on the NeverInclude list, filter applies
+	if isDomainExcluded {
+		return true
+	}
+
+	// default to not filter the connection pair
+	return false
+}
+
+func (fs *Filter) CheckIfInternal(host net.IP) bool {
+	return util.ContainsIP(fs.InternalSubnets, host)
+}

config/filter_test.go

@@ -0,0 +1,280 @@
+package config
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilterConnPair(t *testing.T) {
+	internalSubnetListEmpty := []*net.IPNet{}
+
+	internalSubnetList := []*net.IPNet{
+		{IP: net.IP{11, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{120, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	alwaysIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{35, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{170, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	neverIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{12, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{150, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	// set config filter for external to internal to false
+	cfg.Filter.FilterExternalToInternal = false
+
+	// AlwaysInclude list tests
+	t.Run("AlwaysInclude list tests", func(t *testing.T) {
+		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+		checkCases := cfg.Filter.FilterConnPair(net.IP{35, 0, 0, 0}, net.IP{190, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+
+		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+		checkCases = cfg.Filter.FilterConnPair(net.IP{190, 0, 0, 0}, net.IP{35, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// NeverInclude list tests
+	t.Run("NeverInclude list tests", func(t *testing.T) {
+		cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+		checkCases := cfg.Filter.FilterConnPair(net.IP{12, 0, 0, 0}, net.IP{190, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+
+		cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+		checkCases = cfg.Filter.FilterConnPair(net.IP{190, 0, 0, 0}, net.IP{12, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+	})
+
+	// InternalSubnets tests
+	t.Run("InternalSubnets tests", func(t *testing.T) {
+		cfg.Filter.InternalSubnets = internalSubnetList
+
+		// Both are external
+		checkCases := cfg.Filter.FilterConnPair(net.IP{185, 0, 0, 0}, net.IP{16, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+
+		// Both are internal
+		checkCases = cfg.Filter.FilterConnPair(net.IP{11, 0, 0, 0}, net.IP{120, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+
+		// Source is external, destination is internal, FilterExternalToInternal set
+		cfg.Filter.FilterExternalToInternal = true
+		checkCases = cfg.Filter.FilterConnPair(net.IP{180, 0, 0, 0}, net.IP{11, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{11, 0, 0, 0}, net.IP{120, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+
+		// Empty list
+		cfg.Filter.InternalSubnets = internalSubnetListEmpty
+		checkCases = cfg.Filter.FilterConnPair(net.IP{180, 0, 0, 0}, net.IP{80, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+}
+
+func TestFilterDNSPair(t *testing.T) {
+	internalSubnetListEmpty := []*net.IPNet{}
+
+	internalSubnetList := []*net.IPNet{
+		{IP: net.IP{11, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{120, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	alwaysIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{35, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{170, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	neverIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{12, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{150, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	// set config filter for external to internal to false
+	cfg.Filter.FilterExternalToInternal = false
+
+	// AlwaysInclude list tests
+	t.Run("AlwaysInclude list tests", func(t *testing.T) {
+		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+		checkCases := cfg.Filter.FilterDNSPair(net.IP{35, 0, 0, 0}, net.IP{190, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{190, 0, 0, 0}, net.IP{35, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// NeverInclude list tests
+	t.Run("NeverInclude list tests", func(t *testing.T) {
+		cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+		checkCases := cfg.Filter.FilterDNSPair(net.IP{12, 0, 0, 0}, net.IP{190, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+		cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{190, 0, 0, 0}, net.IP{12, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+	})
+
+	// InternalSubnets tests
+	t.Run("InternalSubnets tests", func(t *testing.T) {
+		cfg.Filter.InternalSubnets = internalSubnetList
+
+		// Both are external
+		checkCases := cfg.Filter.FilterDNSPair(net.IP{185, 0, 0, 0}, net.IP{16, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+
+		// Source is external, destination is internal, FilterExternalToInternal set
+		cfg.Filter.FilterExternalToInternal = true
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{180, 0, 0, 0}, net.IP{120, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{11, 0, 0, 0}, net.IP{120, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+
+		// Empty list
+		cfg.Filter.InternalSubnets = internalSubnetListEmpty
+		checkCases = cfg.Filter.FilterDNSPair(net.IP{180, 0, 0, 0}, net.IP{80, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+}
+
+func TestFilterSingleIP(t *testing.T) {
+	alwaysIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{35, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{170, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	neverIncludedSubnetList := []*net.IPNet{
+		{IP: net.IP{12, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{150, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	// AlwaysInclude list test
+	t.Run("AlwaysInclude list test", func(t *testing.T) {
+		cfg.Filter.AlwaysIncludedSubnets = alwaysIncludedSubnetList
+		checkCases := cfg.Filter.FilterSingleIP(net.IP{35, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// NeverInclude list test
+	t.Run("NeverInclude list test", func(t *testing.T) {
+		cfg.Filter.NeverIncludedSubnets = neverIncludedSubnetList
+		checkCases := cfg.Filter.FilterSingleIP(net.IP{12, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+	})
+}
+
+func TestFilterDomain(t *testing.T) {
+	alwaysIncludedDomainList := []string{
+		"trustmebro-university.com",
+		"expressbuy.com",
+	}
+
+	neverIncludedDomainList := []string{
+		"bing.com",
+		"google.com",
+	}
+
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	// AlwaysInclude list test
+	t.Run("AlwaysInclude list test", func(t *testing.T) {
+		cfg.Filter.AlwaysIncludedDomains = alwaysIncludedDomainList
+		checkCases := cfg.Filter.FilterDomain("trustmebro-university.com")
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// NeverInclude list test
+	t.Run("NeverInclude list test", func(t *testing.T) {
+		cfg.Filter.NeverIncludedDomains = neverIncludedDomainList
+		checkCases := cfg.Filter.FilterDomain("bing.com")
+		require.True(t, checkCases, "filter state should match expected value")
+	})
+}
+
+func TestFilterNeverInclude(t *testing.T) {
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	t.Run("Value not in NeverInclude list", func(t *testing.T) {
+		filtered := cfg.Filter.FilterSingleIP(net.IP{65, 0, 0, 0})
+		require.False(t, filtered, "filter state should match expected value")
+	})
+
+	t.Run("IPv4 broadcast", func(t *testing.T) {
+		filtered := cfg.Filter.FilterSingleIP(net.IPv4bcast)
+		require.True(t, filtered, "filter state should match expected value")
+	})
+
+	t.Run("IPv4 all zeros address", func(t *testing.T) {
+		filtered := cfg.Filter.FilterSingleIP(net.IPv4zero)
+		require.True(t, filtered, "filter state should match expected value")
+	})
+
+	t.Run("IPv6 unspecified address", func(t *testing.T) {
+		filtered := cfg.Filter.FilterSingleIP(net.IPv6unspecified)
+		require.True(t, filtered, "filter state should match expected value")
+	})
+}
+
+func TestCheckIfInternal(t *testing.T) {
+	internalSubnetList := []*net.IPNet{
+		{IP: net.IP{11, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+		{IP: net.IP{120, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}},
+	}
+
+	// load config
+	cfg, err := getDefaultConfig()
+	require.NoError(t, err)
+
+	// set internal subnets
+	cfg.Filter.InternalSubnets = internalSubnetList
+
+	// internal ip
+	t.Run("Valid Internal IP", func(t *testing.T) {
+		checkCases := cfg.Filter.CheckIfInternal(net.IP{11, 0, 0, 0})
+		require.True(t, checkCases, "filter state should match expected value")
+	})
+
+	// external ip
+	t.Run("Valid External IP", func(t *testing.T) {
+		checkCases := cfg.Filter.CheckIfInternal(net.IP{110, 0, 0, 0})
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// unspecified ip
+	t.Run("Unspecified IPv6", func(t *testing.T) {
+		checkCases := cfg.Filter.CheckIfInternal(net.IPv6unspecified)
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// all zeros ip
+	t.Run("All Zeros IPv4", func(t *testing.T) {
+		checkCases := cfg.Filter.CheckIfInternal(net.IPv4zero)
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+	// broadcast ip
+	t.Run("Broadcast IP", func(t *testing.T) {
+		checkCases := cfg.Filter.CheckIfInternal(net.IPv4bcast)
+		require.False(t, checkCases, "filter state should match expected value")
+	})
+
+}

database/analysis_tables.go

@@ -0,0 +1,478 @@
+package database
+
+import (
+	"context"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+)
+
+func (db *DB) createThreatMixtapeTable(ctx context.Context) error {
+	err := db.Conn.Exec(ctx, `--sql
+		CREATE TABLE IF NOT EXISTS {database:Identifier}.threat_mixtape (
+			analyzed_at DateTime64(6),
+			import_id FixedString(16),
+			hash FixedString(16),
+			src IPv6,
+			dst IPv6,
+			src_nuid UUID,
+			dst_nuid UUID,
+			fqdn String,
+			server_ips Array(IPv6),
+			proxy_ips Array(IPv6),
+			total_bytes UInt64,
+			last_seen DateTime(),
+			port_proto_service Array(String),
+
+			-- counts
+			count UInt64,
+			ts_unique UInt64,
+			proxy_count UInt64,
+			open_count UInt64,
+
+			-- c2 over dns connection info
+			direct_conns Array(IPv6),
+			queried_by Array(IPv6),
+
+			-- **** THREAT INDICATORS ****
+			-- BEACONING
+			beacon_type LowCardinality(String),
+			beacon_score Float32,
+			beacon_threat_score Float32,
+			ts_score Float32,
+			ds_score Float32,
+			dur_score Float32,
+			hist_score Float32,
+			ts_intervals Array(Int64),
+			ts_interval_counts Array(Int64),
+			ds_sizes Array(Int64),
+			ds_size_counts Array(Int64),
+			
+			-- LONG CONNECTIONS
+			total_duration Float64,
+			long_conn_score Float32,
+
+			-- STROBE
+			strobe_score Float32,
+
+			-- C2 OVER DNS
+			subdomain_count UInt64,
+			c2_over_dns_score Float32,
+			c2_over_dns_direct_conn_score Float32,
+
+			-- THREAT INTEL
+			threat_intel Bool,
+			threat_intel_score Float32,
+
+			-- **** MODIFIERS ****
+			modifier_name LowCardinality(String),
+			modifier_score Float32,
+			modifier_value String,
+
+			-- PREVALENCE
+			prevalence_total UInt64,
+			prevalence Float32,
+			prevalence_score Float32,
+
+			first_seen_historical DateTime(),
+			first_seen_score Float32,
+
+			-- THREAT INTEL DATA SIZE
+			threat_intel_data_size_score Float32,
+
+
+			-- MISSING HOST HEADER
+			missing_host_count UInt64,
+			missing_host_header_score Float32
+
+		) ENGINE = MergeTree()
+		PRIMARY KEY (analyzed_at, dst_nuid, src_nuid, src, fqdn, dst, hash)
+		ORDER BY (analyzed_at, dst_nuid, src_nuid, src, fqdn, dst, hash)
+	`)
+	return err
+}
+
+func (db *DB) createHistoricalFirstSeenMaterializedViews(ctx context.Context) error {
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_conn_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				if(src_local = true, dst, src) as ip,
+				'' as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.conn
+		GROUP BY (fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_openconn_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				if(src_local = true, dst, src) as ip,
+				'' as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.openconn
+		GROUP BY (fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_ssl_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				'::' as ip,
+				server_name as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.ssl
+		GROUP BY ( fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_openssl_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				'::' as ip,
+				server_name as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.openssl
+		GROUP BY ( fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_http_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				'::' as ip,
+				host as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.http
+		GROUP BY ( fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_openhttp_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				'::' as ip,
+				host as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.openhttp
+		GROUP BY ( fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.historical_first_seen_dns_mv
+		TO metadatabase.historical_first_seen AS
+			SELECT
+				'::' as ip,
+				query as fqdn,
+				minSimpleState(ts) as first_seen,
+				maxSimpleState(ts) as last_seen
+		FROM {database:Identifier}.dns
+		GROUP BY ( fqdn, ip)
+	`); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (db *DB) createMIMETypeURIsTable(ctx context.Context) error {
+
+	err := db.Conn.Exec(ctx, `--sql
+		CREATE TABLE IF NOT EXISTS {database:Identifier}.mime_type_uris (
+			import_hour DateTime(),
+			hour DateTime(),
+			hash FixedString(16),
+			uri String,
+			path String,
+			extension String,
+			mime_type String,
+			mismatch_count AggregateFunction(count, UInt64),
+		)
+		ENGINE = AggregatingMergeTree()
+		PRIMARY KEY (hour, hash, uri)
+	`)
+
+	if err != nil {
+		return err
+	}
+	// This view is used to detect MIME type/URI mismatches
+	// If a HTTP connection's MIME type matches a MIME type in the metadatabase.valid_mime_types table
+	// and its extension does not match the associated values for that MIME type, then it should be added to this table
+	err = db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.mime_type_uris_mv
+		TO {database:Identifier}.mime_type_uris AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			hash,
+			uri,
+			path(uri) as path,
+			-- get the extension from the path
+			CASE
+				-- if the path does not contain a . or ends with a ., then the extension is an empty string
+				WHEN position(reverse(arrayElement(splitByChar('/', path), -1)), '.') = 0 OR endsWith(path, '.') THEN ''
+				-- otherwise, split the last segment by . and take the last element as the extension
+				ELSE splitByChar('.', arrayElement(splitByChar('/', path), -1))[-1]
+			END AS extension,
+			dst_mime_types as mime_type,
+			countState() AS mismatch_count
+		FROM {database:Identifier}.http h
+		-- for each uri, get the extension and join it with the valid mime types, 
+		-- keeping only the rows where the extension does not match the valid extension
+	    ARRAY JOIN dst_mime_types
+		LEFT SEMI JOIN metadatabase.valid_mime_types v ON dst_mime_types = v.mime_type
+		WHERE uri != '/' AND extension != v.extension
+		GROUP BY import_hour, hour, hash, uri, path, extension, mime_type
+	`)
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (db *DB) createRareSignatureTable(ctx context.Context) error {
+
+	err := db.Conn.Exec(ctx, `--sql
+		CREATE TABLE IF NOT EXISTS {database:Identifier}.rare_signatures (
+			import_hour DateTime(),
+			hour DateTime(),
+			src IPv6,
+			src_nuid UUID,
+			signature String,
+			is_ja3 Bool,
+			times_used_dst AggregateFunction(uniqExact, IPv6),
+			times_used_fqdn AggregateFunction(uniqExact, String)
+		)
+		ENGINE = AggregatingMergeTree()
+		PRIMARY KEY (src_nuid, src, signature )
+	`)
+
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.rare_signatures_http_mv
+		TO {database:Identifier}.rare_signatures AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			src,
+			src_nuid,
+			useragent as signature,
+			false as is_ja3,
+			uniqExactState(dst) as times_used_dst,
+			uniqExactState(host) as times_used_fqdn
+		FROM {database:Identifier}.http
+		WHERE length(useragent) > 0
+		GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3)
+	`)
+
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.rare_signatures_ssl_mv
+		TO {database:Identifier}.rare_signatures AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			src,
+			src_nuid,
+			ja3 as signature,
+			true as is_ja3,
+			uniqExactState(dst) as times_used_dst,
+			uniqExactState(server_name) as times_used_fqdn
+		FROM {database:Identifier}.ssl
+		WHERE length(ja3) > 0
+		GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3)
+	`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.rare_signatures_missing_host_mv
+		TO {database:Identifier}.rare_signatures AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			src,
+			src_nuid,
+			missing_host_useragent as signature,
+			false as is_ja3,
+			uniqExactState(if(src_local, dst, src)) as times_used_dst
+		FROM {database:Identifier}.conn
+		WHERE length(missing_host_useragent) > 0 AND missing_host_header = true
+		GROUP BY (import_hour, hour, src, src_nuid, signature, is_ja3)
+	`)
+	if err != nil {
+		return err
+	}
+
+	return err
+
+}
+
+func (db *DB) createPortInfoTable(ctx context.Context) error {
+
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE TABLE IF NOT EXISTS {database:Identifier}.port_info (
+			import_hour DateTime(),
+			hour DateTime(),
+			hash FixedString(16),
+			src IPv6,
+			src_nuid UUID,
+			dst IPv6,
+			dst_nuid UUID,
+			fqdn String,
+			dst_port UInt16,
+			proto LowCardinality(String),
+			service LowCardinality(String),
+			icmp_type UInt16,
+			icmp_code UInt16,
+			conn_state LowCardinality(String),
+			count AggregateFunction(count, UInt64),
+			bytes_sent AggregateFunction(sum, Int64),
+			bytes_received AggregateFunction(sum, Int64)
+		)
+		ENGINE = AggregatingMergeTree()
+		PRIMARY KEY (hour, hash, dst_port, proto, service, icmp_type, icmp_code)
+	`); err != nil {
+		return err
+	}
+
+	// conn
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.port_info_ip_mv
+		TO {database:Identifier}.port_info AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			hash,
+			src,
+			src_nuid,
+			dst,
+			dst_nuid,
+			dst_port,
+			proto,
+			service,
+			if(proto = 'icmp', src_port, 0) as icmp_type,
+			if(proto = 'icmp', dst_port, 0) as icmp_code,
+			conn_state,
+			countState() as count,
+			sumState(src_ip_bytes) as bytes_sent,
+			sumState(dst_ip_bytes) as bytes_received
+		FROM {database:Identifier}.conn
+		WHERE missing_host_header = false
+		GROUP BY (import_hour, hour, hash, src, src_nuid, dst, dst_nuid, dst_port, proto, service, icmp_type, icmp_code, conn_state)
+	`); err != nil {
+		return err
+	}
+
+	// http
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.port_info_http_mv
+		TO {database:Identifier}.port_info AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			hash,
+			src,
+			src_nuid,
+			host as fqdn,
+			dst_port,
+			proto,
+			service,
+			conn_state,
+			countStateIf(multi_request = false) as count, -- only count unique zeek_uids, not each multi-request
+			sumState(src_ip_bytes) as bytes_sent,
+			sumState(dst_ip_bytes) as bytes_received
+		FROM {database:Identifier}.http
+		GROUP BY (import_hour, hour, hash, src, src_nuid, fqdn, dst_port, proto, service, conn_state)
+	`); err != nil {
+		return err
+	}
+
+	// ssl
+	if err := db.Conn.Exec(ctx, `--sql
+		CREATE MATERIALIZED VIEW IF NOT EXISTS {database:Identifier}.port_info_ssl_mv
+		TO {database:Identifier}.port_info AS
+		SELECT
+			toStartOfHour(import_time) as import_hour,
+			toStartOfHour(ts) as hour,
+			hash,
+			src,
+			src_nuid,
+			server_name as fqdn,
+			dst_port,
+			proto,
+			service,
+			conn_state,
+			countState() as count,
+			sumState(src_ip_bytes) as bytes_sent,
+			sumState(dst_ip_bytes) as bytes_received
+		FROM {database:Identifier}.ssl
+		GROUP BY (import_hour, hour, hash, src, src_nuid, fqdn, dst_port, proto, service, conn_state)
+	`); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (db *DB) createSensorDBAnalysisTables() error {
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+
+	err := db.createThreatMixtapeTable(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = db.createRareSignatureTable(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = db.createMIMETypeURIsTable(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = db.createPortInfoTable(ctx)
+	if err != nil {
+		return err
+	}
+
+	// only create historical first seen mvs for rolling datasets
+	if db.Rolling {
+		err = db.createHistoricalFirstSeenMaterializedViews(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

database/db.go

@@ -0,0 +1,396 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/logger"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	driver "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+var ErrInvalidDatabaseConnection = fmt.Errorf("database connection is nil")
+var ErrInvalidMinMaxTimestamp = fmt.Errorf("invalid min or max timestamp")
+
+// DB is the workhorse container for messing with the database
+type DB struct {
+	Conn            driver.Conn
+	selected        string
+	Rolling         bool
+	rebuild         bool
+	ctx             context.Context
+	cancel          context.CancelFunc
+	ImportStartedAt time.Time
+}
+
+// GetSelectedDB returns the name of the target database of db connection
+func (db *DB) GetSelectedDB() string {
+	return db.selected
+}
+
+// QueryParameters generates ClickHouse query parameters by creating a context with the specified parameters in it
+func (db *DB) QueryParameters(params clickhouse.Parameters) context.Context {
+	return clickhouse.Context(db.ctx, clickhouse.WithParameters(params))
+}
+
+// GetContext returns the context for the database connection
+func (db *DB) GetContext() context.Context {
+	return db.ctx
+}
+
+// getConn returns the driver connection
+func (db *DB) getConn() driver.Conn {
+	return db.Conn
+}
+
+func (db *DB) GetBeaconMinMaxTimestamps() (time.Time, time.Time, bool, error) {
+
+	var minTS, maxTS time.Time
+	var notFromConn bool
+
+	if db.Conn == nil {
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, ErrInvalidDatabaseConnection
+	}
+
+	logger := logger.GetLogger()
+
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+	// min timestamp: max timestamp - 24 hours, capped to the actual minimum timestamp from the logs
+	// max timestamp: max timestamp in the logs
+	err := db.Conn.QueryRow(ctx, `
+		SELECT greatest(min_ts, timestamp_sub(HOUR, 24, max_ts)) as min_ts, max_ts FROM (
+			SELECT min(min_ts) AS min_ts, max(max_ts) AS max_ts FROM metadatabase.min_max
+			WHERE database = {database:String} AND beacon = true
+			GROUP BY database
+		)
+	`).Scan(&minTS, &maxTS)
+
+	// return error if the error is not a no rows found error
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		logger.Err(err).Str("database", db.selected).Msg("failed to get max ts from metadatabase min_max table")
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, err
+	}
+
+	if maxTS.IsZero() {
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, ErrInvalidMinMaxTimestamp
+	}
+	if minTS.IsZero() {
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, ErrInvalidMinMaxTimestamp
+	}
+
+	// if dataset is not rolling or if the max timestamp is over 24 hours ago, use the max timestamp
+	return minTS, maxTS, notFromConn, nil
+
+}
+
+func (db *DB) GetTrueMinMaxTimestamps() (time.Time, time.Time, bool, bool, error) {
+	logger := logger.GetLogger()
+
+	var minTS, maxTS time.Time
+	var notFromConn bool
+	var useCurrentTime bool
+
+	if db.Conn == nil {
+		return time.Unix(0, 0), time.Unix(0, 0), false, false, ErrInvalidDatabaseConnection
+	}
+
+	rolling, err := GetRollingStatus(db.GetContext(), db.Conn, db.GetSelectedDB())
+	if err != nil && !errors.Is(err, ErrDatabaseNotFound) {
+		return time.Unix(0, 0), time.Unix(0, 0), false, false, err
+	}
+	if errors.Is(err, ErrDatabaseNotFound) {
+		rolling = db.Rolling
+	}
+
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+	// min timestamp: max timestamp - 24 hours, capped to the actual minimum timestamp from the logs
+	// max timestamp: max timestamp in the logs
+	err = db.Conn.QueryRow(ctx, `
+		SELECT greatest(min_ts, timestamp_sub(HOUR, 24, max_ts)) as min_ts, max_ts FROM (
+			SELECT min(min_ts) AS min_ts, max(max_ts) AS max_ts FROM metadatabase.min_max
+			WHERE database = {database:String} 
+			GROUP BY database
+		)
+	`).Scan(&minTS, &maxTS)
+
+	// return error if the error is not a no rows found error
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		logger.Err(err).Str("database", db.selected).Msg("failed to get max ts from metadatabase min_max table")
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, useCurrentTime, err
+	}
+
+	if maxTS.IsZero() {
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, useCurrentTime, fmt.Errorf("could not find any viable max timestamp")
+	}
+	if minTS.IsZero() {
+		return time.Unix(0, 0), time.Unix(0, 0), notFromConn, useCurrentTime, fmt.Errorf("could not find any viable min timestamp")
+	}
+
+	// if dataset is rolling and the max timestamp is not over 24 hours ago, use the current time for first seen
+	if rolling && time.Since(maxTS).Hours() <= 24 {
+		useCurrentTime = true
+	}
+
+	// if dataset is not rolling or if the max timestamp is over 24 hours ago, use the max timestamp
+	return minTS, maxTS, notFromConn, useCurrentTime, nil
+
+}
+
+// GetNetworkSize returns the number of distinct internal hosts for the past 24 hours, which is used to determine prevalence
+func (db *DB) GetNetworkSize(minTS time.Time) (uint64, error) {
+	logger := logger.GetLogger()
+
+	var networkSize uint64
+
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"min_ts": fmt.Sprintf("%d", minTS.UTC().Unix()),
+	})
+
+	err := db.Conn.QueryRow(ctx, `
+		SELECT count() FROM (
+			-- uconn
+			SELECT DISTINCT src FROM uconn
+			WHERE src_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT dst AS src FROM uconn
+			WHERE dst_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			-- openconn
+			SELECT DISTINCT src FROM openconn
+			WHERE src_local = true
+			UNION DISTINCT
+			SELECT DISTINCT dst AS src FROM openconn
+			WHERE dst_local = true
+			UNION DISTINCT
+			-- http
+			SELECT DISTINCT src FROM usni
+			WHERE http = true AND src_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT dst AS src FROM usni
+			WHERE http = true AND dst_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			-- openhttp
+			SELECT DISTINCT src FROM openhttp
+			WHERE src_local = true
+			UNION DISTINCT
+			SELECT DISTINCT dst AS src FROM openhttp
+			WHERE dst_local = true
+			UNION DISTINCT
+			-- dns
+			SELECT DISTINCT src FROM udns
+			WHERE src_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+			UNION DISTINCT
+			SELECT DISTINCT dst AS src FROM udns
+			WHERE dst_local = true AND hour >= toStartOfHour(fromUnixTimestamp({min_ts:Int64}))
+		)
+	`).Scan(&networkSize)
+
+	if err != nil {
+		logger.Err(err).Str("database", db.selected).Msg("failed to network size from uconn table")
+		return networkSize, err
+	}
+
+	return networkSize, nil
+}
+
+// TruncateTmpLinkTables truncates the tables that are used to link zeek uids.
+// This should be called after each import so that these tmp tables don't take up unnecessary disk space.
+func (db *DB) TruncateTmpLinkTables() error {
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.conn_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.ssl_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.http_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openconn_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openssl_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openhttp_tmp
+	`); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ResetTemporaryTables clears out data in tmp tables (if they exist) from the previous import
+func (db *DB) ResetTemporaryTables() error {
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openconn
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openhttp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openssl
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.uconn_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.openconnhash_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.opensniconn_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.sniconn_tmp
+	`); err != nil {
+		return err
+	}
+
+	if err := db.Conn.Exec(ctx, `--sql
+		TRUNCATE TABLE IF EXISTS {database:Identifier}.dns_tmp
+	`); err != nil {
+		return err
+	}
+
+	return db.TruncateTmpLinkTables()
+
+}
+
+// ConnectToDB sets up a new connection to the specified database
+func ConnectToDB(ctx context.Context, db string, cfg *config.Config, cancel context.CancelFunc) (*DB, error) {
+	// connect to the database
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.DBConnection},
+		Auth: clickhouse.Auth{
+			Database: db,
+			Username: "default",
+			Password: "",
+		},
+		DialContext: func(ctx context.Context, addr string) (net.Conn, error) {
+			// dialCount++
+			var d net.Dialer
+			return d.DialContext(ctx, "tcp", addr)
+		},
+		Debug: false,
+		Debugf: func(format string, v ...any) {
+			log.Println(format, v)
+		},
+		Settings: clickhouse.Settings{
+			"max_execution_time": cfg.MaxQueryExecutionTime,
+			"mutations_sync":     1,
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+		DialTimeout:          time.Second * 120,
+		MaxOpenConns:         50,
+		MaxIdleConns:         50,
+		ConnMaxLifetime:      time.Duration(1) * time.Hour,
+		ConnOpenStrategy:     clickhouse.ConnOpenInOrder,
+		BlockBufferSize:      10,
+		MaxCompressionBuffer: 10240,
+
+		ClientInfo: clickhouse.ClientInfo{ // optional, please see Client info section in the clickhouse-go README.md
+			Products: []struct {
+				Name    string
+				Version string
+			}{
+				{Name: "rita", Version: "0.1"},
+			},
+		},
+	})
+
+	// check if the connection call had any errors
+	if err != nil {
+		return nil, err
+	}
+
+	// check if the connection is valid
+	if err := conn.Ping(ctx); err != nil {
+		// if exception, ok := err.(*clickhouse.Exception); ok {
+		// 	fmt.Printf("Exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
+		// }
+		return nil, err
+	}
+
+	// fmt.Println("Validated connection to database", db)
+
+	return &DB{
+		Conn:     conn,
+		ctx:      ctx,
+		cancel:   cancel,
+		selected: db,
+	}, nil
+}
+
+// GetFirstSeenTimestamp gets the relative timestamp to use for calculating/displaying first seen.
+// Returns max timestamp, whether or not to use the current time, and error
+// func (db *DB) GetFirstSeenTimestamp() (time.Time, time.Time, bool, error) {
+// 	rolling, err := GetRollingStatus(db.GetContext(), db.Conn, db.GetSelectedDB())
+// 	if err != nil {
+// 		return time.Unix(0, 0), time.Unix(0, 0), false, err
+// 	}
+
+// 	minTS, maxTS, _, err := db.GetMinMaxTimestamps()
+// 	if err != nil {
+// 		return time.Unix(0, 0), time.Unix(0, 0), false, fmt.Errorf("could not get min/max timestamps for analysis: %w", err)
+// 	}
+
+// 	// if dataset is not rolling or if the max timestamp is over 24 hours ago, use the max timestamp
+// 	if !rolling || time.Since(maxTS).Hours() > 24 {
+// 		return maxTS, minTS, false, nil
+// 	}
+
+// 	// if rolling and maxTS <= 24 hrs ago, use the current time
+// 	return time.Unix(0, 0), time.Unix(0, 0), true, nil
+// }

database/db_test.go

@@ -0,0 +1,795 @@
+package database_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/database"
+	"activecm/rita/util"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func (d *DatabaseTestSuite) TestConnectToDB() {
+
+	// create and connect to a new database
+	d.Run("Connect to Existing Database", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to createddatabase should not produce an error")
+		require.NotNil(t, db)
+	})
+
+	// attempt to connect to a non-existent database
+	d.Run("Connect to Non-Existent Database", func() {
+		t := d.T()
+		db, err := database.ConnectToDB(context.Background(), "nonExistentDB", d.cfg, nil)
+		require.Error(t, err, "connecting to a non-existent database should produce an error")
+		require.Nil(t, db)
+	})
+
+	// attempt to connect with invalid configuration
+	d.Run("Invalid Configuration", func() {
+		t := d.T()
+		invalidCfg := *d.cfg
+		invalidCfg.DBConnection = "invalid connection string"
+
+		db, err := database.ConnectToDB(context.Background(), "testDB", &invalidCfg, nil)
+		require.Error(t, err, "connecting with invalid configuration should produce an error")
+		require.Nil(t, db)
+	})
+
+	// attempt to connect with a cancelled context
+	d.Run("Cancel Context", func() {
+		t := d.T()
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel()
+
+		// attempt to connect with a cancelled context
+		db, err := database.ConnectToDB(ctx, "testDB", d.cfg, nil)
+		require.Error(t, err, "connecting with a cancelled context should produce an error")
+		require.Nil(t, db)
+	})
+}
+
+func (d *DatabaseTestSuite) TestMinMaxTimestamps() {
+	d.Run("24 Hour Dataset", func() {
+		t := d.T()
+		// import a dataset with 24 hours of data
+		results, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the min and max timestamps from a test function that queries the conn, openconn, and dns tables
+		_, max := getMinMaxTimestampsFromTables(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching beacon min and max timestamps should not produce an error")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min and max timestamps should not produce an error")
+
+		// validate that the current time is not used
+		require.False(t, useCurrentTime, "useCurrentTime should be false")
+
+		// check that the timestamps are within the expected range
+		require.True(t, maxTS.After(minTS), "max timestamp should be after min timestamp")
+
+		// capped min timestamp
+		minTSCapped := max.Add(-24 * time.Hour)
+		require.InDelta(t, 24.0, maxTS.Sub(minTS).Hours(), 0.1, "timestamp difference should be close to 24 hours")
+
+		fmt.Println("Max Timestamp: ", maxTS)
+		fmt.Println("Min Timestamp: ", minTS)
+
+		// check that the true timestamps match the min and max timestamps from the test function
+		require.Equal(t, minTSCapped, minTS, "min timestamp should match min timestamp from the test function")
+		require.Equal(t, max, maxTS, "max timestamp should match max timestamp from the test function")
+
+		// check that the beacon timestamps are within the expected range
+		require.True(t, maxTSBeacon.Before(max) || maxTSBeacon.Equal(max), "max beacon timestamp should be before or equal to max timestamp from the test function")
+		require.True(t, minTSBeacon.After(minTSCapped) || minTSBeacon.Equal(minTSCapped), "min beacon timestamp should be after or equal to min timestamp from the test function")
+	})
+
+	d.Run("Less Than 24 Hours of Data", func() {
+		t := d.T()
+		// import a dataset with < 24 hours of data
+		results, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/proxy_rolling", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the min and max timestamps from a test function that queries the conn, openconn, and dns tables
+		min, max := getMinMaxTimestampsFromTables(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching beacon min and max timestamps should not produce an error")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min and max timestamps should not produce an error")
+
+		// validate that the current time is not used
+		require.False(t, useCurrentTime, "useCurrentTime should be false")
+
+		// check that the timestamps are within the expected range
+		require.True(t, maxTS.After(minTS), "max timestamp should be after min timestamp")
+		require.Less(t, maxTS.Sub(minTS).Hours(), 24.0, "timestamp difference should be less than 24 hours")
+
+		// check that the max timestamp matches the max timestamps from the test function
+		require.Equal(t, max, maxTS, "max timestamp should match max timestamp from test function")
+
+		// since the dataset is less than 24 hours, the min timestamp should match the min timestamp from the test function
+		require.Equal(t, min, minTS, "min timestamp should match min timestamp from test function")
+
+		// check that the beacon timestamps are within the expected range
+		require.True(t, maxTSBeacon.Before(max) || maxTSBeacon.Equal(max), "max beacon timestamp should be before or equal to max timestamp from the test function")
+		require.True(t, minTSBeacon.After(min) || minTSBeacon.Equal(min), "min beacon timestamp should be after or equal to min timestamp from the test function")
+	})
+
+	d.Run("Greater Than 24 Hours of Data", func() {
+		t := d.T()
+
+		afs := afero.NewMemMapFs()
+
+		// create directory
+		directory := "/logs"
+		err := afs.Mkdir(directory, os.FileMode(0o775))
+		require.NoError(t, err, "creating directory should not produce an error")
+
+		// create mock data file
+		fileName := "conn.log"
+		path := filepath.Join(directory, fileName)
+		file, err := afs.Create(path)
+		require.NoError(t, err, "creating file should not produce an error")
+
+		// set file permissions
+		err = afs.Chmod(path, os.FileMode(0o775))
+		require.NoError(t, err, "changing file permissions should not produce an error")
+
+		// generate timestamp from current time and format to the timestamp format used in the log file
+		currentTime := time.Now().UTC()
+		formattedMaxTime := fmt.Sprintf("%d.%06d", currentTime.Unix(), currentTime.Nanosecond()/1000)
+
+		// create interval timestamps of 4 hours ago
+		fourHoursAgo := currentTime.Add(-4 * time.Hour)
+		formattedTime := fmt.Sprintf("%d.%06d", fourHoursAgo.Unix(), fourHoursAgo.Nanosecond()/1000)
+
+		// create timestamp for 48 hours ago
+		twoDaysAgo := currentTime.Add(-48 * time.Hour)
+		formattedMinTime := fmt.Sprintf("%d.%06d", twoDaysAgo.Unix(), twoDaysAgo.Nanosecond()/1000)
+
+		// create mock data
+		log := []byte("#separator \\x09\n" +
+			"#set_separator\t,\n" +
+			"#empty_field\t(empty)\n" +
+			"#unset_field\t-\n" +
+			"#path\tconn\n" +
+			"#open\t2019-02-28-12-07-01\n" +
+			"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+			"#types\ttime\tstring\taddr\taddr\n" +
+			formattedMaxTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+			formattedMinTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n",
+		)
+		bytesWritten, err := file.Write(log)
+		require.NoError(t, err, "writing data to file should not produce an error")
+		require.Equal(t, len(log), bytesWritten, "number of bytes written should be equal to the length of the log data")
+
+		err = file.Close()
+		require.NoError(t, err, "closing file should not produce an error")
+
+		// import the mock data
+		results, err := cmd.RunImportCmd(time.Now(), d.cfg, afs, directory, "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the min and max timestamps from a test function that queries the conn, openconn, and dns tables
+		min, max := getMinMaxTimestampsFromTables(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching beacon min and max timestamps should not produce an error")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min and max timestamps should not produce an error")
+
+		// validate that the current time is not used
+		require.False(t, useCurrentTime, "useCurrentTime should be false")
+
+		// check that the timestamps are within the expected range
+		require.True(t, maxTS.After(minTS), "max timestamp should be after min timestamp")
+		require.InDelta(t, 24.0, maxTS.Sub(minTS).Hours(), 0.1, "timestamp difference should be close to 24 hours")
+
+		// check that the max timestamp matches the max timestamps from the test function
+		require.Equal(t, max, maxTS, "max timestamp should match max timestamp from test function")
+
+		// since the dataset is > 24 hours, the min timestamp will get capped to 24 hours from the max timestamp
+		require.NotEqual(t, min, minTS, "min timestamp should not match min timestamp from test function")
+		require.Equal(t, maxTS.Add(-24*time.Hour), minTS, "min timestamp should be 24 hours from max timestamp")
+
+		// check that the beacon timestamps are within the expected range
+		require.True(t, maxTSBeacon.Before(max) || maxTSBeacon.Equal(max), "max beacon timestamp should be before or equal to max timestamp from the test function")
+		require.True(t, minTSBeacon.After(maxTS.Add(-24*time.Hour)) || minTSBeacon.Equal(maxTS.Add(-24*time.Hour)), "min beacon timestamp should be after or equal to min timestamp from the test function")
+	})
+
+	d.Run("Rolling, Max Timestamp < 24Hrs Ago", func() {
+		t := d.T()
+		afs := afero.NewMemMapFs()
+
+		// create directory
+		directory := "/logs"
+		err := afs.Mkdir(directory, os.FileMode(0o775))
+		require.NoError(t, err, "creating directory should not produce an error")
+
+		// create mock data file
+		fileName := "conn.log"
+		path := filepath.Join(directory, fileName)
+		file, err := afs.Create(path)
+		require.NoError(t, err, "creating file should not produce an error")
+
+		// set file permissions
+		err = afs.Chmod(path, os.FileMode(0o775))
+		require.NoError(t, err, "changing file permissions should not produce an error")
+
+		// generate timestamp from current time and format to the timestamp format used in the log file
+		maxTime := time.Now().UTC().Add(-1 * time.Hour)
+		formattedMaxTime := fmt.Sprintf("%d.%06d", maxTime.Unix(), maxTime.Nanosecond()/1000)
+
+		// create interval timestamps of 4 hours ago
+		fourHoursAgo := maxTime.Add(-4 * time.Hour)
+		formattedTime := fmt.Sprintf("%d.%06d", fourHoursAgo.Unix(), fourHoursAgo.Nanosecond()/1000)
+
+		// create timestamp for 24 hours ago
+		minTime := maxTime.Add(-24 * time.Hour)
+		formattedMinTime := fmt.Sprintf("%d.%06d", minTime.Unix(), minTime.Nanosecond()/1000)
+
+		// create mock data
+		log := []byte("#separator \\x09\n" +
+			"#set_separator\t,\n" +
+			"#empty_field\t(empty)\n" +
+			"#unset_field\t-\n" +
+			"#path\tconn\n" +
+			"#open\t2019-02-28-12-07-01\n" +
+			"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+			"#types\ttime\tstring\taddr\taddr\n" +
+			formattedMaxTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+			formattedMinTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n",
+		)
+		bytesWritten, err := file.Write(log)
+		require.NoError(t, err, "writing data to file should not produce an error")
+		require.Equal(t, len(log), bytesWritten, "number of bytes written should be equal to the length of the log data")
+
+		err = file.Close()
+		require.NoError(t, err, "closing file should not produce an error")
+
+		// import the mock data
+		results, err := cmd.RunImportCmd(time.Now(), d.cfg, afs, directory, "testDB", true, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the min and max timestamps from a test function that queries the conn, openconn, and dns tables
+		min, max := getMinMaxTimestampsFromTables(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching beacon min and max timestamps should not produce an error")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min and max timestamps should not produce an error")
+
+		// validate that the current time should be used since the max timestamp is <= 24 hours ago and the dataset is rolling
+		require.True(t, useCurrentTime, "useCurrentTime should be true")
+
+		// check that the timestamps are within the expected range
+		require.True(t, maxTS.After(minTS), "max timestamp should be after min timestamp")
+		require.InDelta(t, 24.0, maxTS.Sub(minTS).Hours(), 0.1, "timestamp difference should be close to 24 hours")
+
+		// check that the timestamps match the min and max timestamps from the test function
+		require.Equal(t, min, minTS, "min timestamp should match min timestamp from the test function")
+		require.Equal(t, max, maxTS, "max timestamp should match max timestamp from the test function")
+
+		// check that the beacon timestamps are within the expected range
+		require.True(t, maxTSBeacon.Before(max) || maxTSBeacon.Equal(max), "max beacon timestamp should be before or equal to max timestamp from the test function")
+		require.True(t, minTSBeacon.After(min) || minTSBeacon.Equal(min), "min beacon timestamp should be after or equal to min timestamp from the test function")
+	})
+
+	d.Run("Rolling, Max Timestamp > 24Hrs Ago", func() {
+		t := d.T()
+		afs := afero.NewMemMapFs()
+
+		// create directory
+		directory := "/logs"
+		err := afs.Mkdir(directory, os.FileMode(0o775))
+		require.NoError(t, err, "creating directory should not produce an error")
+
+		// create mock data file
+		fileName := "conn.log"
+		path := filepath.Join(directory, fileName)
+		file, err := afs.Create(path)
+		require.NoError(t, err, "creating file should not produce an error")
+
+		// set file permissions
+		err = afs.Chmod(path, os.FileMode(0o775))
+		require.NoError(t, err, "changing file permissions should not produce an error")
+
+		// generate timestamp from current time and format to the timestamp format used in the log file
+		maxTime := time.Now().UTC().Add(-25 * time.Hour)
+		formattedMaxTime := fmt.Sprintf("%d.%06d", maxTime.Unix(), maxTime.Nanosecond()/1000)
+
+		// create interval timestamps of 4 hours ago
+		fourHoursAgo := maxTime.Add(-4 * time.Hour)
+		formattedTime := fmt.Sprintf("%d.%06d", fourHoursAgo.Unix(), fourHoursAgo.Nanosecond()/1000)
+
+		// create timestamp for 48 hours ago
+		minTime := maxTime.Add(-48 * time.Hour)
+		formattedMinTime := fmt.Sprintf("%d.%06d", minTime.Unix(), minTime.Nanosecond()/1000)
+
+		// create mock data
+		log := []byte("#separator \\x09\n" +
+			"#set_separator\t,\n" +
+			"#empty_field\t(empty)\n" +
+			"#unset_field\t-\n" +
+			"#path\tconn\n" +
+			"#open\t2019-02-28-12-07-01\n" +
+			"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+			"#types\ttime\tstring\taddr\taddr\n" +
+			formattedMaxTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+			formattedMinTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n",
+		)
+		bytesWritten, err := file.Write(log)
+		require.NoError(t, err, "writing data to file should not produce an error")
+		require.Equal(t, len(log), bytesWritten, "number of bytes written should be equal to the length of the log data")
+
+		err = file.Close()
+		require.NoError(t, err, "closing file should not produce an error")
+
+		// import the mock data
+		results, err := cmd.RunImportCmd(time.Now(), d.cfg, afs, directory, "testDB", true, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the min and max timestamps from a test function that queries the conn, openconn, and dns tables
+		min, max := getMinMaxTimestampsFromTables(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching beacon min and max timestamps should not produce an error")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min and max timestamps should not produce an error")
+
+		// validate that the current time is not used
+		require.False(t, useCurrentTime, "useCurrentTime should be false")
+
+		// check that the timestamps are within the expected range
+		require.True(t, maxTS.After(minTS), "max timestamp should be after min timestamp")
+		require.InDelta(t, 24.0, maxTS.Sub(minTS).Hours(), 0.1, "timestamp difference should be close to 24 hours")
+
+		// check that the max timestamp matches the max timestamps from the test function
+		require.Equal(t, max, maxTS, "max timestamp should match max timestamp from test function")
+
+		// since the dataset is > 24 hours, the min timestamp will get capped to 24 hours from the max timestamp
+		require.NotEqual(t, min, minTS, "min timestamp should not match min timestamp from test function")
+		require.Equal(t, maxTS.Add(-24*time.Hour), minTS, "min timestamp should be 24 hours from max timestamp")
+
+		// check that the beacon timestamps are within the expected range
+		require.True(t, maxTSBeacon.Before(max) || maxTSBeacon.Equal(max), "max beacon timestamp should be before or equal to max timestamp from the test function")
+		require.True(t, minTSBeacon.After(maxTS.Add(-24*time.Hour)) || minTSBeacon.Equal(maxTS.Add(-24*time.Hour)), "min beacon timestamp should be after or equal to min timestamp from the test function")
+	})
+
+	d.Run("Non-Existent Database / Invalid Connection", func() {
+		t := d.T()
+		db := database.DB{}
+
+		// fetch the beacon minimum and maximum timestamps
+		minTSBeacon, maxTSBeacon, _, err := db.GetBeaconMinMaxTimestamps()
+		require.Error(t, err, "fetching beacon min and max timestamps should produce an error")
+		require.Equal(t, minTSBeacon, time.Unix(0, 0), "max timestamp should be zero unix time")
+		require.Equal(t, maxTSBeacon, time.Unix(0, 0), "min timestamp should be zero unix time")
+		require.Equal(t, err, database.ErrInvalidDatabaseConnection, "error should be invalid database connection")
+
+		// fetch the true minimum and maximum timestamps
+		minTS, maxTS, _, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.Error(t, err, "fetching min and max timestamps should produce an error")
+
+		require.Equal(t, maxTS, time.Unix(0, 0), "max timestamp should be zero unix time")
+		require.Equal(t, minTS, time.Unix(0, 0), "min timestamp should be zero unix time")
+		require.False(t, useCurrentTime, "value passed back to useCurrentTime should be false")
+		require.Equal(t, err, database.ErrInvalidDatabaseConnection, "error should be invalid database connection")
+
+	})
+}
+
+func getMinMaxTimestampsFromTables(t *testing.T, db *database.DB, importID util.FixedString) (time.Time, time.Time) { // uses valid dataset
+	t.Helper()
+
+	type minMaxRes struct {
+		Min time.Time `ch:"min_timestamp"`
+		Max time.Time `ch:"max_timestamp"`
+	}
+
+	var result minMaxRes
+
+	// set context with importID and database parameters
+	ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"importID": importID.Hex(),
+		"database": db.GetSelectedDB(),
+	}))
+
+	// get min and max timestamps from the conn, openconn, and udns tables
+	err := db.Conn.QueryRow(ctx, `
+			SELECT MIN(ts) AS min_timestamp, MAX(ts) AS max_timestamp
+			FROM (
+				SELECT ts FROM {database:Identifier}.conn
+				UNION ALL
+				SELECT ts FROM {database:Identifier}.openconn
+				UNION ALL
+				SELECT hour as ts FROM {database:Identifier}.udns
+			) AS combined_timestamps
+		`).ScanStruct(&result)
+	require.NoError(t, err)
+
+	fmt.Println("Min Timestamp: ", result.Min)
+	fmt.Println("Max Timestamp: ", result.Max)
+
+	return result.Min, result.Max
+}
+
+func (d *DatabaseTestSuite) TestGetNetworkSize() {
+	d.Run("Valid TSV Dataset", func() {
+		t := d.T()
+		// import a dataset
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		var result struct {
+			Count uint64 `ch:"count"`
+		}
+
+		// verify http linking wrote no more than 20 records with the same zeek uid
+		err = db.Conn.QueryRow(db.GetContext(), `
+				SELECT count() as count FROM conn
+			`).ScanStruct(&result)
+		require.NoError(t, err, "querying conn table should not produce an error")
+
+		fmt.Println("Count: ", result.Count)
+
+		// get the min timestamp
+		minTS, _, notFromConn, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching min timestamp should not produce an error")
+
+		// validate which table min max is from
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+		// get the network size (number of unique internal hosts for the past 24 hours)
+		networkSize, err := db.GetNetworkSize(minTS)
+		require.NoError(t, err, "getting network size should not produce an error")
+
+		// verify the expected network size
+		require.Greater(t, networkSize, uint64(0), "network size should be greater than zero")
+
+		require.Equal(t, 15, int(networkSize), "network size for valid tsv dataset should match expected value")
+	})
+
+	d.Run("Mocked Log File with Duplicate Internal Hosts", func() {
+		t := d.T()
+		afs := afero.NewMemMapFs()
+
+		// create directory
+		directory := "/logs"
+		err := afs.Mkdir(directory, os.FileMode(0o775))
+		require.NoError(t, err, "creating directory should not produce an error")
+
+		// create mock data file
+		fileName := "conn.log"
+		path := filepath.Join(directory, fileName)
+		file, err := afs.Create(path)
+		require.NoError(t, err, "creating file should not produce an error")
+
+		// set file permissions
+		err = afs.Chmod(path, os.FileMode(0o775))
+		require.NoError(t, err, "changing file permissions should not produce an error")
+
+		// create mock data with duplicates and write data to file
+		log := []byte("#separator \\x09\n" +
+			"#set_separator\t,\n" +
+			"#empty_field\t(empty)\n" +
+			"#unset_field\t-\n" +
+			"#path\tconn\n" +
+			"#open\t2019-02-28-12-07-01\n" +
+			"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+			"#types\ttime\tstring\taddr\taddr\n" +
+			"1715640994.367201\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			"1715640994.367201\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			"1715641054.367201\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			"1715641054.367201\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			"1715641114.367201\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			"1715641114.367201\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			"1715641174.367201\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			"1715641174.367201\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			"1715641234.367201\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+			"1715641234.367201\tCxT125\t10.0.0.5\t52.12.0.5\n",
+		)
+		bytesWritten, err := file.Write(log)
+		require.NoError(t, err, "writing data to file should not produce an error")
+		require.Equal(t, len(log), bytesWritten, "number of bytes written should be equal to the length of the log data")
+
+		err = file.Close()
+		require.NoError(t, err, "closing file should not produce an error")
+
+		// import the mock data
+		_, err = cmd.RunImportCmd(time.Now(), d.cfg, afs, directory, "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		var result struct {
+			Count uint64 `ch:"count"`
+		}
+
+		// create a query to count number of conn records
+		err = db.Conn.QueryRow(db.GetContext(), `
+				SELECT count() as count FROM conn
+			`).ScanStruct(&result)
+		require.NoError(t, err, "querying conn table should not produce an error")
+		require.Equal(t, uint64(10), result.Count, "number of records in conn table should be 10")
+
+		// get the min timestamp
+		minTS, _, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching min timestamp should not produce an error")
+
+		// get the network size (number of unique internal hosts for the past 24 hours)
+		networkSize, err := db.GetNetworkSize(minTS)
+		require.NoError(t, err, "getting network size should not produce an error")
+
+		// verify the expected network size
+		fmt.Println("Network Size: ", networkSize)
+		require.Greater(t, networkSize, uint64(0), "network size should be greater than zero")
+		require.Equal(t, uint64(5), networkSize, "network size for test 2 should be match expected value")
+
+		// remove the mock directory
+		require.NoError(t, afs.RemoveAll(directory), "removing mock directory should not produce an error")
+
+	})
+
+	d.Run("Open Conn Dataset", func() {
+		t := d.T()
+		// import a dataset
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/open_conns/open/open_conn.log", "testDBzzzz", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), "testDBzzzz", d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		var result struct {
+			Count uint64 `ch:"count"`
+		}
+
+		// validate that number of conn records is zero
+		err = db.Conn.QueryRow(db.GetContext(), `
+				SELECT count() as count FROM conn
+			`).ScanStruct(&result)
+		require.NoError(t, err, "querying conn table should not produce an error")
+		require.Equal(t, uint64(0), result.Count, "number of records in conn table should be 0")
+
+		// validate that number of open conn records is > 0
+		err = db.Conn.QueryRow(db.GetContext(), `
+				SELECT count() as count FROM openconn
+			`).ScanStruct(&result)
+		require.NoError(t, err, "querying openconn table should not produce an error")
+		require.Greater(t, result.Count, uint64(0), "number of open conn records should be >0")
+
+		// get the min timestamp
+		minTS, _, _, _, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "fetching min timestamp should not produce an error")
+
+		// get the network size (number of unique internal hosts for the past 24 hours)
+		networkSize, err := db.GetNetworkSize(minTS)
+		require.NoError(t, err, "getting network size should not produce an error")
+
+		// verify the expected network size
+		// fmt.Println("Network Size: ", networkSize)
+		require.Greater(t, networkSize, uint64(0), "network size should be greater than zero")
+		require.Equal(t, uint64(12), networkSize, "network size for test should match expected value")
+
+	})
+
+}
+
+// TestGetFirstSeenTimestamp tests the GetFirstSeenTimestamp function
+// exec: go test -count=1 -v ./database -run DatabaseTestSuit/TestGetFirstSeenTimestamp
+func (d *DatabaseTestSuite) TestGetFirstSeenTimestamp() {
+	d.Run("Not Rolling", func() {
+		t := d.T()
+		dbName := "testdb_not_rolling"
+		// import a dataset as non-rolling
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", dbName, false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), dbName, d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the max timestamp
+		_, maxTS, notFromConn, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching max timestamp should not produce an error")
+
+		// validate which table min max is from
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+		// get the first seen timestamp
+		// firstSeen, _, useCurrentTime, err := db.GetFirstSeenTimestamp()
+		// require.NoError(t, err, "getting first seen timestamp should not produce an error")
+		require.True(t, maxTS.After(time.Unix(0, 0)), "first seen timestamp should be after zero unix time")
+		// require.Equal(t, maxTS, firstSeen, "first seen timestamp should be equal to max timestamp")
+	})
+
+	d.Run("Rolling, Max Timestamp > 24Hrs Ago", func() {
+		t := d.T()
+		dbName := "testdb_rolling_old"
+		// import a dataset as rolling
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", dbName, true, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), dbName, d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the max timestamp
+		_, maxTS, notFromConn, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching max timestamp should not produce an error")
+
+		// validate which table min max is from
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+		// get the first seen timestamp
+		// firstSeen, _, useCurrentTime, err := db.GetFirstSeenTimestamp()
+		// require.NoError(t, err, "getting first seen timestamp should not produce an error")
+		require.True(t, maxTS.After(time.Unix(0, 0)), "first seen timestamp should be after zero unix time")
+		// require.Equal(t, maxTS, firstSeen, "first seen timestamp should be equal to max timestamp")
+	})
+
+	d.Run("Rolling, Max Timestamp < 24Hrs Ago", func() {
+		t := d.T()
+
+		afs := afero.NewMemMapFs()
+		dbName := "testdb_rolling_live"
+		// create directory
+		directory := "/logs"
+		err := afs.Mkdir(directory, os.FileMode(0o775))
+		require.NoError(t, err, "creating directory should not produce an error")
+
+		// create mock data file
+		fileName := "conn.log"
+		path := filepath.Join(directory, fileName)
+		file, err := afs.Create(path)
+		require.NoError(t, err, "creating file should not produce an error")
+
+		// set file permissions
+		err = afs.Chmod(path, os.FileMode(0o775))
+		require.NoError(t, err, "changing file permissions should not produce an error")
+
+		// generate timestamp from current time and format to the timestamp format used in the log file
+		currentTime := time.Now().UTC()
+		formattedTime := fmt.Sprintf("%d.%06d", currentTime.Unix(), currentTime.Nanosecond()/1000)
+
+		// create mock data
+		log := []byte("#separator \\x09\n" +
+			"#set_separator\t,\n" +
+			"#empty_field\t(empty)\n" +
+			"#unset_field\t-\n" +
+			"#path\tconn\n" +
+			"#open\t2019-02-28-12-07-01\n" +
+			"#fields\tts\tuid\tid.orig_h\tid.resp_h\n" +
+			"#types\ttime\tstring\taddr\taddr\n" +
+			formattedTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT121\t10.0.0.1\t52.12.0.1\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT122\t10.0.0.2\t52.12.0.2\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT123\t10.0.0.3\t52.12.0.3\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT124\t10.0.0.4\t52.12.0.4\n" +
+			formattedTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n" +
+			formattedTime + "\tCxT125\t10.0.0.5\t52.12.0.5\n",
+		)
+		bytesWritten, err := file.Write(log)
+		require.NoError(t, err, "writing data to file should not produce an error")
+		require.Equal(t, len(log), bytesWritten, "number of bytes written should be equal to the length of the log data")
+
+		err = file.Close()
+		require.NoError(t, err, "closing file should not produce an error")
+
+		// import the mock data as rolling
+		_, err = cmd.RunImportCmd(time.Now(), d.cfg, afs, directory, dbName, true, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		// connect to the database
+		db, err := database.ConnectToDB(context.Background(), dbName, d.cfg, nil)
+		require.NoError(t, err, "connecting to database should not produce an error")
+
+		// get the max timestamp
+		_, maxTS, _, err := db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "fetching max timestamp should not produce an error")
+
+		// validate which table min max is from
+		// require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+		// verify that the max timestamp is set to the current time (which is less than 24 hours ago)
+		require.WithinDuration(t, currentTime, maxTS, time.Second, "max timestamp should be set to the current time")
+
+		// get the first seen timestamp
+		// firstSeen, _, useCurrentTime, err := db.GetFirstSeenTimestamp()
+		// require.NoError(t, err, "getting first seen timestamp should not produce an error")
+
+		// remove the mock directory
+		err = afs.RemoveAll(directory)
+		require.NoError(t, err, "removing mock directory should not produce an error")
+
+	})
+}

database/metadb.go

@@ -0,0 +1,348 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/util"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+)
+
+type MetaDBImportedFile struct {
+	Hash      *util.FixedString `ch:"hash"`
+	ImportID  *util.FixedString `ch:"import_id"`
+	Database  string            `ch:"database"`
+	Timestamp time.Time         `ch:"ts"`
+	Path      string            `ch:"path"`
+}
+
+type MetaDBImportRecord struct {
+	ImportID         *util.FixedString `ch:"import_id"`
+	Rolling          bool              `ch:"rolling"`
+	Database         string            `ch:"database"`
+	Rebuild          bool              `ch:"rebuild"`
+	StartedAt        int64             `ch:"started_at"`
+	EndedAt          time.Time         `ch:"ended_at"`
+	HoursSeen        []time.Time       `ch:"hours_seen"`
+	ImportVersion    string            `ch:"import_version"`
+	MinTimestamp     time.Time         `ch:"min_timestamp"`
+	MaxTimestamp     time.Time         `ch:"max_timestamp"`
+	MinOpenTimestamp time.Time         `ch:"min_open_timestamp"`
+	MaxOpenTimestamp time.Time         `ch:"max_open_timestamp"`
+}
+
+// createMetaDatabase creates the metadatabase and its tables if any part of it doesn't exist
+func (server *ServerConn) createMetaDatabase() error {
+	err := server.Conn.Exec(server.ctx, `
+		CREATE DATABASE IF NOT EXISTS metadatabase
+	`)
+	if err != nil {
+		return err
+	}
+
+	err = server.createMetaDatabaseImportsTable()
+	if err != nil {
+		return err
+	}
+
+	err = server.createMetaDatabaseFilesTable()
+	if err != nil {
+		return err
+	}
+
+	err = server.createMetaDatabaseMinMaxTable()
+	if err != nil {
+		return err
+	}
+
+	err = server.createThreatIntelTables()
+	if err != nil {
+		return err
+	}
+
+	err = server.createValidMIMETypeTable()
+	if err != nil {
+		return err
+	}
+
+	err = server.createHistoricalFirstSeenTable()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// createMetaDatabaseFilesTable creates the metadatabase.files table
+func (server *ServerConn) createMetaDatabaseFilesTable() error {
+	err := server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.files (
+			hash FixedString(16),
+			database String,
+			import_id FixedString(16),
+			rolling Bool,
+			ts DateTime(),
+			path String
+		)
+		ENGINE = MergeTree()
+		PRIMARY KEY (database, import_id, hash, path)
+	`)
+
+	return err
+}
+
+// createMetaDatabaseImportsTable creates the metadatabase.imports table
+func (server *ServerConn) createMetaDatabaseImportsTable() error {
+	err := server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.imports (
+			import_id FixedString(16),
+			rolling Bool,
+			database String,
+			rebuild Bool,
+			-- started_at is measured in Microseconds
+			started_at DateTime64(6), 
+			ended_at DateTime(),
+			hours_seen Array(DateTime()),
+			import_version String,
+			min_timestamp DateTime(),
+			max_timestamp DateTime(),
+			min_open_timestamp DateTime(),
+			max_open_timestamp DateTime()
+		)
+		ENGINE = MergeTree()
+		PRIMARY KEY (database, ended_at, started_at, import_id)
+	`)
+	return err
+}
+
+func (server *ServerConn) createMetaDatabaseMinMaxTable() error {
+	// err := server.Conn.Exec(server.ctx, `--sql
+	// 	CREATE TABLE IF NOT EXISTS metadatabase.min_max_raw (
+	// 		database String,
+	// 		rolling Bool,
+	// 		beacon String,
+	// 		min_ts DateTime(),
+	// 		max_ts DateTime()
+	// 	)
+	// 	ENGINE = MergeTree()
+	// 	PRIMARY KEY (database)
+	// `)
+	// if err != nil {
+	// 	return err
+	// }
+
+	err := server.Conn.Exec(server.ctx, `--sql
+		CREATE TABLE IF NOT EXISTS metadatabase.min_max (
+			database String,
+			rolling Bool,
+			beacon Bool,
+			min_ts SimpleAggregateFunction(min, DateTime()),
+			max_ts SimpleAggregateFunction(max, DateTime())
+		)
+		ENGINE = AggregatingMergeTree()
+		PRIMARY KEY (database, beacon)
+	`)
+	if err != nil {
+		return err
+	}
+
+	// if err := server.Conn.Exec(server.ctx, `--sql
+	// 	CREATE MATERIALIZED VIEW IF NOT EXISTS metadatabase.min_max_default_mv
+	// 	TO metadatabase.min_max AS
+	// 	SELECT
+	// 		database,
+	// 		rolling,
+	// 		beacon,
+	// 		minSimpleState(min_ts) as min_ts,
+	// 		maxSimpleState(max_ts) as max_ts
+	// 	FROM metadatabase.min_max_raw
+	// 	GROUP BY (database, rolling, beacon)
+	// `); err != nil {
+	// 	return err
+	// }
+	return nil
+}
+
+// MarkFileImportedInMetaDB adds the given path to the metadatabase.files table to mark it as being used
+func (db *DB) MarkFileImportedInMetaDB(hash util.FixedString, importID util.FixedString, path string) error {
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"hash":      hash.Hex(),
+		"importID":  importID.Hex(),
+		"database":  db.selected,
+		"timestamp": strconv.FormatInt(time.Now().UTC().Unix(), 10),
+		"path":      path,
+		"rolling":   strconv.FormatBool(db.Rolling),
+	})
+
+	err := db.Conn.Exec(ctx, `
+		INSERT INTO metadatabase.files (hash, import_id, database, rolling, ts, path)
+		VALUES (unhex({hash:String}), unhex({importID:String}), {database:String}, {rolling:Bool}, {timestamp:Int32}, {path:String})
+	`)
+	return err
+}
+
+/* *** TRACKING IMPORTS ***
+Data in ClickHouse is meant to be append-only. This means that we cannot easily update records.
+The metadatabase.imports table acts as a log of events for imports. In order to track the start and completion
+of an import, one record is inserted at the beginning of the import, and another record is imported at the end.
+*/
+
+// AddImportStartRecordToMetaDB inserts a record into the metadatabase.imports table to mark that an import has started
+func (db *DB) AddImportStartRecordToMetaDB(importID util.FixedString) error {
+	if config.Version == "" {
+		return fmt.Errorf("cannot add import record to metadb, version is not set")
+	}
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"importID":        importID.Hex(),
+		"rolling":         strconv.FormatBool(db.Rolling),
+		"database":        db.selected,
+		"rebuild":         strconv.FormatBool(db.rebuild),
+		"importStartedAt": strconv.FormatInt(db.ImportStartedAt.UnixMicro(), 10),
+		"importVersion":   config.Version,
+	})
+
+	err := db.Conn.Exec(ctx, `
+		INSERT INTO metadatabase.imports (import_id, rolling, database, rebuild, started_at)
+		VALUES (unhex({importID:String}), {rolling:Bool}, {database:String}, {rebuild:Bool}, fromUnixTimestamp64Micro({importStartedAt:Int64}))
+	`)
+
+	return err
+}
+
+// AddImportFinishedRecordToMetaDB inserts a record into the metadatabase.imports table to mark that an import has finished
+func (db *DB) AddImportFinishedRecordToMetaDB(importID util.FixedString, minTS, maxTS time.Time) error {
+	// get min and max timestamps from the imported conn logs
+	type minMaxRes struct {
+		Min time.Time `ch:"min_ts"`
+		Max time.Time `ch:"max_ts"`
+	}
+
+	// get min and max timestamps from the imported open_conn logs
+	var minMaxOpen minMaxRes
+
+	err := db.Conn.QueryRow(db.GetContext(), `
+		SELECT min(ts) as min_ts, max(ts) as max_ts FROM openconn
+	`).ScanStruct(&minMaxOpen)
+	if err != nil {
+		return err
+	}
+
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"importID":        importID.Hex(),
+		"rolling":         strconv.FormatBool(db.Rolling),
+		"database":        db.selected,
+		"importStartedAt": strconv.FormatInt(db.ImportStartedAt.UnixMicro(), 10),
+		"importEndedAt":   strconv.FormatInt(time.Now().Unix(), 10),
+		"minTs":           strconv.FormatInt(minTS.UTC().Unix(), 10),
+		"maxTs":           strconv.FormatInt(maxTS.UTC().Unix(), 10),
+		"minOpenTs":       strconv.FormatInt(minMaxOpen.Min.UTC().Unix(), 10),
+		"maxOpenTs":       strconv.FormatInt(minMaxOpen.Max.UTC().Unix(), 10),
+		"importVersion":   config.Version,
+	})
+
+	err = db.Conn.Exec(ctx, `
+		INSERT INTO metadatabase.imports (import_id, rolling, database, started_at, ended_at, min_timestamp, max_timestamp, min_open_timestamp, max_open_timestamp)
+		VALUES (unhex({importID:String}), {rolling:Bool}, {database:String}, fromUnixTimestamp64Micro({importStartedAt:Int64}), fromUnixTimestamp({importEndedAt:Int32}), fromUnixTimestamp({minTs:Int32}), fromUnixTimestamp({maxTs:Int32}), fromUnixTimestamp({minOpenTs:Int32}), fromUnixTimestamp({maxOpenTs:Int32}))
+	`)
+	return err
+}
+
+// CheckIfFilesWereAlreadyImported calls checkFileHashes for each log type
+func (db *DB) CheckIfFilesWereAlreadyImported(fileMap map[string][]string) (int, error) {
+	totalFileCount := 0
+	for logType, logList := range fileMap {
+		results, err := db.checkFileHashes(logList)
+		if err != nil {
+			return totalFileCount, err
+		}
+		fileMap[logType] = results
+		totalFileCount += len(results)
+	}
+
+	return totalFileCount, nil
+}
+
+// checkFileHashes filters fileList to only files that haven't already been imported for this dataset
+func (db *DB) checkFileHashes(fileList []string) ([]string, error) {
+	// format array for clickhouse parameters
+	files := "["
+	for _, file := range fileList {
+		files += fmt.Sprintf("'%s',", file)
+	}
+	files += "]"
+
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+		"files":    files,
+	})
+
+	var importedFiles []struct {
+		Path string `ch:"path"`
+	}
+
+	// query for files in this fileList that have already been imported
+	err := db.Conn.Select(ctx, &importedFiles, `
+		SELECT path FROM metadatabase.files WHERE database = {database:String} AND path IN {files:Array(String)}
+	`)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert imported files array into a map
+	importedFilesMap := make(map[string]bool)
+	for _, file := range importedFiles {
+		importedFilesMap[file.Path] = true
+	}
+
+	var nonImportedFiles []string
+
+	// build a list of files that haven't been imported
+	for _, file := range fileList {
+		if !importedFilesMap[file] {
+			nonImportedFiles = append(nonImportedFiles, file)
+		}
+	}
+
+	return nonImportedFiles, err
+}
+
+// ClearMetaDBEntriesForDatabase deletes all file and import record entries in the metadatabase for the specified database
+func (server *ServerConn) ClearMetaDBEntriesForDatabase(database string) error {
+	// verify that the metadatabase exists
+	exists, err := DatabaseExists(server.Conn, server.ctx, "metadatabase")
+	if err != nil {
+		return err
+	}
+
+	// clear the imported files and min_max records for the specified database if metadatabase exists
+	if exists {
+		if err := server.clearImportedFilesFromMetaDB(database); err != nil {
+			return err
+		}
+
+		if err := server.clearDatabaseFromMetaDB(database); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// clearImportedFilesFromMetaDB deletes entries in files table for specified database
+func (server *ServerConn) clearImportedFilesFromMetaDB(database string) error {
+	ctx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": database}))
+	err := server.Conn.Exec(ctx, `
+		DELETE FROM metadatabase.files WHERE database = {database:String}
+	`, database)
+	return err
+}
+
+func (server *ServerConn) clearDatabaseFromMetaDB(database string) error {
+	ctx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": database}))
+	err := server.Conn.Exec(ctx, `
+		DELETE FROM metadatabase.min_max WHERE database = {database:String}
+	`, database)
+	return err
+}

database/server.go

@@ -0,0 +1,478 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/logger"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	driver "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/spf13/afero"
+)
+
+type ServerConn struct {
+	Conn   driver.Conn
+	addr   string
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+var ErrNoMetaDBImportRecordForDatabase = errors.New("no import record found for database")
+var ErrDatabaseNotFound = errors.New("database does not exist")
+var ErrDatabaseNameEmpty = errors.New("database name cannot be empty")
+var ErrMissingConfig = errors.New("config cannot be nil")
+var errImportTwiceNonRolling = errors.New("cannot import more than once to a non-rolling database")
+var errRollingStatusFailure = errors.New("failed to detect rolling status of given import database")
+var errRollingFlagMissing = errors.New("cannot import non-rolling data to a rolling database")
+
+// SetUpNewImport creates the database requested for this import and returns a new DB struct for connection to said database
+func SetUpNewImport(afs afero.Fs, cfg *config.Config, dbName string, rollingFlag bool, rebuildFlag bool) (*DB, error) {
+	logger := logger.GetLogger()
+
+	// validate parameters
+	if cfg == nil {
+		return nil, ErrMissingConfig
+	}
+
+	if dbName == "" {
+		return nil, ErrDatabaseNameEmpty
+	}
+
+	ctx := context.Background()
+
+	// connect to ClickHouse server
+	server, err := ConnectToServer(ctx, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// set up metadatabase if it does not exist yet
+	err = server.createServerDBTables()
+	if err != nil {
+		return nil, err
+	}
+
+	err = server.createMetaDatabaseTTLs(cfg.MonthsToKeepHistoricalFirstSeen)
+	if err != nil {
+		return nil, err
+	}
+
+	// drop database if rebuild flag was passed
+	if rebuildFlag {
+		err = server.DeleteSensorDB(dbName)
+		if err != nil {
+			return nil, err
+		}
+		logger.Info().Str("database", dbName).Msg("Successfully rebuilt import database")
+	}
+
+	rolling, err := server.checkRolling(dbName, rollingFlag, rebuildFlag)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := server.createSensorDatabase(cfg, dbName, rolling)
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.ResetTemporaryTables()
+	if err != nil {
+		return nil, err
+	}
+
+	err = server.syncThreatIntelFeedsFromConfig(afs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	err = server.importValidMIMETypes(afs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	// err = server.ParseHints(afs, cfg)
+	// if err != nil {
+	// 	return nil, err
+	// }
+
+	// // set rolling flag
+	db.Rolling = rollingFlag
+
+	// set rebuild flag
+	db.rebuild = rebuildFlag
+
+	return db, nil
+
+}
+
+// QueryParameters generates ClickHouse query parameters by creating a context with the specified parameters in it
+func (server *ServerConn) QueryParameters(params clickhouse.Parameters) context.Context {
+	return clickhouse.Context(server.ctx, clickhouse.WithParameters(params))
+}
+
+// GetContext returns the context for the database connection
+func (server *ServerConn) GetContext() context.Context {
+	return server.ctx
+}
+
+// getConn returns the driver connection
+func (server *ServerConn) getConn() driver.Conn {
+	return server.Conn
+}
+
+func (server *ServerConn) createHistoricalFirstSeenTable() error {
+	err := server.Conn.Exec(context.Background(), `--sql
+		CREATE TABLE IF NOT EXISTS metadatabase.historical_first_seen (
+			ip IPv6,
+			fqdn String,
+			first_seen SimpleAggregateFunction(min, DateTime()),
+			last_seen SimpleAggregateFunction(max, DateTime())
+		) ENGINE = AggregatingMergeTree()
+		PRIMARY KEY (fqdn, ip)
+	`)
+	return err
+}
+
+// createSensorDatabase creates a database for the specified sensor and returns a connection to it
+func (server *ServerConn) createSensorDatabase(cfg *config.Config, dbName string, rolling bool) (*DB, error) {
+	logger := logger.GetLogger()
+
+	// create a database named after the specified sensor
+	ctx := clickhouse.Context(context.Background(), clickhouse.WithParameters(clickhouse.Parameters{
+		"database": dbName,
+	}))
+
+	err := server.Conn.Exec(ctx, "CREATE DATABASE IF NOT EXISTS {database:Identifier}")
+	if err != nil {
+		logger.Err(err).Str("database", dbName).
+			Str("database connection", cfg.DBConnection).
+			Msg("failed to create sensor database")
+		return nil, err
+	}
+
+	// connect to newly created database
+	db, err := ConnectToDB(server.ctx, dbName, cfg, server.cancel)
+	if err != nil {
+		logger.Err(err).Str("database", dbName).
+			Str("database connection", cfg.DBConnection).
+			Msg("failed to connect to sensor database")
+		return nil, err
+	}
+
+	// set rolling flag
+	db.Rolling = rolling
+
+	// create tables for the newly created database
+	err = db.createSensorDBTables()
+	if err != nil {
+		logger.Err(err).Str("database", dbName).
+			Str("database connection", cfg.DBConnection).
+			Msg("failed to create tables for import database")
+		return nil, err
+	}
+	// create analysis tables for the newly created database
+	err = db.createSensorDBAnalysisTables()
+	if err != nil {
+		logger.Err(err).Str("database", dbName).
+			Str("database connection", cfg.DBConnection).
+			Msg("failed to create analysis tables for import database")
+		return nil, err
+	}
+
+	// if the database is rolling, create the necessary TTLs on the tables for cleanup
+	if db.Rolling {
+		if err := db.createLogTableTTLs(); err != nil {
+			return nil, err
+		}
+
+		if err := db.createSnapshotTableTTLs(); err != nil {
+			return nil, err
+		}
+	}
+
+	return db, nil
+}
+
+// DropMultipleSensorDatabases drops the databases that match the specified wildcard
+// a wildcard can be in the beginning, end, or both
+func (server *ServerConn) DropMultipleSensorDatabases(dbName string, wildcardStart, wildcardEnd bool) (int, error) {
+	var query string
+	// switch {
+	// case wildcardStart && !wildcardEnd:
+	// 	query = "SHOW DATABASES LIKE '%{database:String}'"
+	// case !wildcardStart && wildcardEnd:
+	// 	query = "SHOW DATABASES LIKE '{database:String}%'"
+	// case wildcardStart && wildcardEnd:
+	// 	query = "SHOW DATABASES LIKE '%{database:String}%'"
+	// case !wildcardStart && !wildcardEnd:
+	// 	return 0, errors.New("no wildcard specified for deleting multiple datasets")
+	// }
+
+	// create query to get the databases that match the wildcard
+	switch {
+	case wildcardStart && wildcardEnd:
+		query = fmt.Sprintf("SHOW DATABASES LIKE '%%%s%%'", dbName)
+	case wildcardStart:
+		query = fmt.Sprintf("SHOW DATABASES LIKE '%%%s'", dbName)
+	case wildcardEnd:
+		query = fmt.Sprintf("SHOW DATABASES LIKE '%s%%'", dbName)
+	default:
+		return 0, errors.New("no wildcard specified for deleting multiple datasets")
+	}
+
+	// execute the query
+	paramsCtx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": dbName}))
+	rows, err := server.Conn.Query(paramsCtx, query)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	// create a counter to keep track of the number of databases deleted
+	var numDeleted int
+
+	// iterate over the databases that match the wildcard
+	for rows.Next() {
+		// get the database name
+		var foundDB string
+		err := rows.Scan(&foundDB)
+		if err != nil {
+			return numDeleted, err
+		}
+
+		// drop the database
+		err = server.DeleteSensorDB(foundDB)
+		if err != nil {
+			return numDeleted, err
+		}
+
+		// increment the number of databases deleted
+		numDeleted++
+	}
+
+	return numDeleted, nil
+}
+
+// dropSensorDatabase drops the specified sensor database
+func (server *ServerConn) dropSensorDatabase(dbName string) error {
+	logger := logger.GetLogger()
+	err := dropDatabase(server.ctx, server.Conn, dbName)
+	if err != nil {
+		logger.Err(err).Str("database", dbName).
+			Msg("failed to drop database")
+		return err
+	}
+	return nil
+}
+
+// DeleteSensorDB deletes the specified database along with its associated imported files in metadatabase.files
+func (server *ServerConn) DeleteSensorDB(database string) error {
+	// drop the database
+	if err := server.dropSensorDatabase(database); err != nil {
+		return err
+	}
+
+	// clear entries in metadatabase
+	if err := server.ClearMetaDBEntriesForDatabase(database); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// GetRollingStatus gets the rolling status of a database
+func GetRollingStatus(dbCtx context.Context, conn driver.Conn, dbName string) (bool, error) {
+	var result struct {
+		Rolling bool `ch:"rolling"`
+	}
+
+	// if import database does not exist, return an error
+	exists, err := SensorDatabaseExists(conn, dbCtx, dbName)
+	if err != nil {
+		return false, err
+	}
+	if !exists {
+		return false, ErrDatabaseNotFound
+	}
+
+	// check the rolling status by looking at the most recent rebuild
+	ctx := clickhouse.Context(dbCtx, clickhouse.WithParameters(clickhouse.Parameters{"database": dbName}))
+	err = conn.QueryRow(ctx, `
+			SELECT rolling FROM metadatabase.min_max WHERE database = {database:String}
+			ORDER BY max_ts DESC
+			LIMIT 1
+	`).ScanStruct(&result)
+
+	if err != nil {
+		return false, err
+	}
+
+	return result.Rolling, nil
+}
+
+// checkRolling checks the rolling status of a database
+func (server *ServerConn) checkRolling(dbName string, rollingFlag bool, rebuildFlag bool) (bool, error) {
+	logger := logger.GetLogger()
+
+	// get the current rolling status of the database from the imports table (if db already exists)
+	rolling, err := GetRollingStatus(server.ctx, server.Conn, dbName)
+
+	switch {
+	// if database doesn't exist, just return the desired rolling status from flag
+	case errors.Is(err, ErrDatabaseNotFound) || errors.Is(err, sql.ErrNoRows):
+		return rollingFlag, nil
+
+	// error executing query
+	case err != nil:
+		logger.Err(err).Str("database", dbName).
+			Str("database connection", server.addr).
+			Msg(errRollingStatusFailure.Error())
+		return rolling, errRollingStatusFailure
+
+	// command is requesting to import data as rolling, but dataset is not rolling
+	case rollingFlag && !rolling && !rebuildFlag:
+		logger.Warn().Str("database", dbName).
+			Msg(errImportTwiceNonRolling.Error())
+		return rolling, errImportTwiceNonRolling
+
+	// command is requesting to import data as non-rolling, but dataset is rolling
+	case rolling && !rollingFlag && !rebuildFlag:
+		logger.Warn().Str("database", dbName).
+			Msg(errRollingFlagMissing.Error())
+		return rolling, errRollingFlagMissing
+	}
+
+	return rolling, nil
+}
+
+type ImportDatabase struct {
+	Name    string    `ch:"database"`
+	Rolling bool      `ch:"rolling"`
+	MinTS   time.Time `ch:"min_ts"`
+	MaxTS   time.Time `ch:"max_ts"`
+}
+
+func (server *ServerConn) ListImportDatabases() ([]ImportDatabase, error) {
+	logger := logger.GetLogger()
+
+	// if metadatabase does not exist, return an empty list
+	exists, err := DatabaseExists(server.Conn, server.ctx, "metadatabase")
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, nil
+	}
+
+	var sensorDBs []ImportDatabase
+
+	// return list of databases based on min_max table
+	query := `
+		SELECT database, rolling, greatest(min_ts, timestamp_sub(WEEK, 2, max_ts)) as min_ts, max_ts FROM (
+			SELECT database, rolling, min(min_ts) AS min_ts, max(max_ts) AS max_ts FROM metadatabase.min_max
+			GROUP BY database, rolling
+			ORDER BY max_ts DESC
+		)
+    `
+	err = server.Conn.Select(server.ctx, &sensorDBs, query)
+	if err != nil {
+		logger.Err(err).Str("database connection", server.addr).Msg("failed to execute import database list query")
+		return nil, err
+	}
+
+	return sensorDBs, nil
+}
+
+func SensorDatabaseExists(conn driver.Conn, ctx context.Context, dbName string) (bool, error) {
+	logger := logger.GetLogger()
+	// check if database actually exists
+	dbExists, err := DatabaseExists(conn, ctx, dbName)
+	if err != nil {
+		logger.Err(err).Str("database", dbName).Msg("failed to check if database exists")
+		return false, err
+	}
+	if !dbExists {
+		return false, nil
+	}
+
+	// check if database is listed in metadatabase
+	paramsCtx := clickhouse.Context(ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": dbName}))
+
+	var exists uint64
+	err = conn.QueryRow(paramsCtx, "SELECT count() FROM metadatabase.min_max WHERE database = {database:String}").Scan(&exists)
+	if err != nil {
+		logger.Err(err).Str("database", dbName).Msg("failed to check if database exists in metadatabase")
+		return false, err
+	}
+	return exists > 0, nil
+}
+
+func GetFlatDatabaseList(dbs []ImportDatabase) []string {
+	var dbList []string
+	for _, db := range dbs {
+		dbList = append(dbList, db.Name)
+	}
+	return dbList
+}
+
+func DatabaseExists(conn driver.Conn, ctx context.Context, dbName string) (bool, error) {
+	logger := logger.GetLogger()
+
+	paramsCtx := clickhouse.Context(ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": dbName}))
+
+	var exists uint64
+	err := conn.QueryRow(paramsCtx, "SELECT count() FROM system.databases WHERE name = {database:String}").Scan(&exists)
+	if err != nil {
+		logger.Err(err).Str("database", dbName).Msg("failed to check if database exists")
+		return false, err
+	}
+
+	return exists > 0, nil
+}
+
+// dropDatabase drops the specified database
+func dropDatabase(ctx context.Context, conn driver.Conn, dbName string) error {
+	paramsCtx := clickhouse.Context(ctx, clickhouse.WithParameters(clickhouse.Parameters{"database": dbName}))
+	err := conn.Exec(paramsCtx, "DROP DATABASE IF EXISTS {database:Identifier}")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ConnectToServer connects to the clickhouse server as the default user
+func ConnectToServer(ctx context.Context, cfg *config.Config) (*ServerConn, error) {
+	logger := logger.GetLogger()
+
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.DBConnection}, // read from env instead
+		Auth: clickhouse.Auth{
+			Database: "default",
+			Username: "default",
+			Password: "",
+		},
+	})
+
+	if err != nil {
+		logger.Err(err).Str("database", "default").
+			Str("database connection", cfg.DBConnection).
+			Msg("failed to connect to ClickHouse server")
+		return nil, err
+	}
+
+	// ping the server to verify connection
+	if err := conn.Ping(ctx); err != nil {
+		return nil, err
+	}
+
+	return &ServerConn{
+		Conn: conn,
+		addr: cfg.DBConnection,
+		ctx:  ctx,
+	}, nil
+}

database/server_test.go

@@ -0,0 +1,413 @@
+package database_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/joho/godotenv"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"github.com/testcontainers/testcontainers-go"
+	cl "github.com/testcontainers/testcontainers-go/modules/clickhouse"
+)
+
+const ConfigPath = "../integration/test_config.hjson"
+
+const TestDataPath = "../test_data"
+
+type DatabaseTestSuite struct {
+	suite.Suite
+	cfg                  *config.Config
+	clickhouseContainer  *cl.ClickHouseContainer
+	clickhouseConnection string
+	server               *database.ServerConn
+}
+
+func TestMain(m *testing.M) {
+	// load environment variables with panic prevention
+	if err := godotenv.Overload("../.env", "../integration/test.env"); err != nil {
+		log.Fatalf("Error loading .env file: %v", err)
+	}
+	os.Exit(m.Run())
+}
+
+func TestDatabase(t *testing.T) {
+	suite.Run(t, new(DatabaseTestSuite))
+}
+
+// SetupSuite is run once before the first test starts
+func (d *DatabaseTestSuite) SetupSuite() {
+	t := d.T()
+
+	// load the config file
+	cfg, err := config.LoadConfig(afero.NewOsFs(), ConfigPath)
+	require.NoError(t, err, "config should load without error")
+
+	// start clickhouse container
+	d.SetupClickHouse(t)
+
+	// update the config to use the clickhouse container connection
+	cfg.DBConnection = d.clickhouseConnection
+
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "config should update without error")
+	d.cfg = cfg
+
+	// connect to clickhouse server
+	server, err := database.ConnectToServer(context.Background(), d.cfg)
+	require.NoError(t, err, "connecting to server should not produce an error")
+	d.server = server
+}
+
+// TearDownSuite is run once after all tests have finished
+func (d *DatabaseTestSuite) TearDownSuite() {
+	if err := d.clickhouseContainer.Terminate(context.Background()); err != nil {
+		log.Fatalf("failed to terminate clickhouse container: %s", err)
+	}
+}
+
+// SetupTest is run before each test method
+// func (d *DatabaseTestSuite) SetupTest() {}
+
+// TearDownTest is run after each test method
+// func (d *DatabaseTestSuite) TearDownTest() {}
+
+// SetupSubTest is run before each subtest
+func (d *DatabaseTestSuite) SetupSubTest() {
+	t := d.T()
+	// fmt.Println("Running setup subtest...")
+
+	// drop all databases that may have been created during subtest
+	if d.server != nil && d.server.Conn != nil {
+		dbs, err := d.server.ListImportDatabases()
+		require.NoError(t, err, "listing databases should not produce an error")
+		for _, db := range dbs {
+			err := d.server.DeleteSensorDB(db.Name)
+			require.NoError(t, err, "dropping database should not produce an error")
+		}
+	}
+}
+
+// TearDownSubTest is run after each subtest
+// func (d *DatabaseTestSuite) TearDownSubTest() {}
+
+// SetupClickHouse creates a ClickHouse container using the test.docker-compose.yml and handles taking it down when complete
+func (d *DatabaseTestSuite) SetupClickHouse(t *testing.T) {
+	t.Helper()
+
+	// get ClickHouse version from environment
+	version := os.Getenv("CLICKHOUSE_VERSION")
+	require.NotEmpty(t, version, "CLICKHOUSE_VERSION environment variable must be set")
+
+	// create ClickHouse container
+	ctx := context.Background()
+	clickHouseContainer, err := cl.RunContainer(ctx,
+		testcontainers.WithImage(fmt.Sprintf("clickhouse/clickhouse-server:%s-alpine", version)),
+		cl.WithUsername("default"),
+		cl.WithPassword(""),
+		cl.WithDatabase("default"),
+		cl.WithConfigFile(filepath.Join("../deployment/", "config.xml")),
+	)
+	require.NoError(t, err, "failed to start clickHouse container")
+
+	// get connection host
+	connectionHost, err := clickHouseContainer.ConnectionHost(ctx)
+	require.NoError(t, err, "failed to get clickHouse connection host")
+
+	// set container and connection host
+	d.clickhouseContainer = clickHouseContainer
+	d.clickhouseConnection = connectionHost
+
+}
+
+func (d *DatabaseTestSuite) TestConnectToServer() {
+
+	// connect to the server with valid configuration
+	d.Run("Successful Server Connection", func() {
+		t := d.T()
+		server, err := database.ConnectToServer(context.Background(), d.cfg)
+		require.NoError(t, err, "connecting to clickhouse server should not produce an error")
+		require.NotNil(t, server, "server connection object should not be nil")
+
+		// ping to ensure the connection is valid
+		err = server.Conn.Ping(context.Background())
+		require.NoError(t, err, "pinging clickhouse server should not produce an error")
+	})
+
+	// attempt to connect with invalid configuration
+	d.Run("Failed Server Connection", func() {
+		t := d.T()
+		invalidCfg := *d.cfg
+		invalidCfg.DBConnection = "invalid connection string"
+
+		server, err := database.ConnectToServer(context.Background(), &invalidCfg)
+		require.Error(t, err, "connecting with invalid configuration should produce an error")
+		require.Nil(t, server, "server connection object should be nil on failed connection")
+	})
+}
+
+func (d *DatabaseTestSuite) TestDeleteSensorDB() {
+	d.Run("Drop Existing Database", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		db, err := database.ConnectToDB(context.Background(), "testDB", d.cfg, nil)
+		require.NoError(t, err, "connecting to created database should not produce an error")
+		require.NotNil(t, db)
+
+		// drop the database
+		err = d.server.DeleteSensorDB("testDB")
+		require.NoError(t, err, "dropping database should not produce an error")
+
+		d.checkDatabaseDeletion("testDB")
+	})
+
+	d.Run("Drop Non-Existent Database", func() {
+		t := d.T()
+
+		// attempt to drop a database that doesn't exist
+		err := d.server.DeleteSensorDB("nonExistentDB")
+		require.NoError(t, err, "attempting to drop a non-existent database should not produce an error")
+	})
+
+}
+
+func (d *DatabaseTestSuite) checkDatabaseDeletion(dbName string) {
+	t := d.T()
+	// attempt to connect to the dropped database
+	db, err := database.ConnectToDB(context.Background(), dbName, d.cfg, nil)
+	require.Error(t, err, "connecting to a dropped database should produce an error")
+	require.Nil(t, db)
+	ctx := d.server.QueryParameters(clickhouse.Parameters{
+		"database": dbName,
+	})
+	// check for db in min_max
+	var count uint64
+	err = d.server.Conn.QueryRow(ctx, `
+		SELECT count() FROM metadatabase.min_max
+		WHERE database = {database:String}
+	`).Scan(&count)
+	require.NoError(t, err, "querying metadatabase.min_max should not produce an error")
+	require.EqualValues(t, 0, count, "there should be no records for a deleted dataset in metadatabase.min_max, database: %s", dbName)
+	// check for db in files
+	err = d.server.Conn.QueryRow(ctx, `
+		SELECT count() FROM metadatabase.files
+		WHERE database = {database:String}
+	`).Scan(&count)
+	require.NoError(t, err, "querying metadatabase.files should not produce an error")
+	require.EqualValues(t, 0, count, "there should be no records for a deleted dataset in metadatabase.files, database: %s", dbName)
+}
+
+func (d *DatabaseTestSuite) checkDatabaseNonDeletion(dbName string) {
+	t := d.T()
+	// attempt to connect to the database
+	db, err := database.ConnectToDB(context.Background(), dbName, d.cfg, nil)
+	require.NoError(t, err, "connecting to a database that was not dropped should not produce an error")
+	require.NotNil(t, db)
+	ctx := d.server.QueryParameters(clickhouse.Parameters{
+		"database": dbName,
+	})
+	// check for db in min_max
+	var count uint64
+	err = d.server.Conn.QueryRow(ctx, `
+		SELECT count() FROM metadatabase.min_max
+		WHERE database = {database:String}
+	`).Scan(&count)
+	require.NoError(t, err, "querying metadatabase.min_max should not produce an error")
+	require.Greater(t, count, uint64(0), "there should be at least 1 record for a dataset in metadatabase.min_max, database: %s", dbName)
+	// check for db in files
+	err = d.server.Conn.QueryRow(ctx, `
+		SELECT count() FROM metadatabase.files
+		WHERE database = {database:String}
+	`).Scan(&count)
+	require.NoError(t, err, "querying metadatabase.files should not produce an error")
+	require.Greater(t, count, uint64(0), "there should be at least 1 record for a dataset in metadatabase.files, database: %s", dbName)
+}
+
+func (d *DatabaseTestSuite) TestDropMultipleSensorDatabases() {
+	// helper function to create databases with various prefixes and suffixes
+	databases := []string{"bingbong", "prefix_bingbong", "bingbong123", "prefix_bingbong123"}
+
+	createDatabases := func(t *testing.T) {
+		t.Helper()
+
+		for _, dbName := range databases {
+			// create a database name with no prefix or suffix
+			_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", dbName, false, false)
+			require.NoError(t, err, "importing data should not produce an error")
+
+		}
+	}
+
+	d.Run("Drop Databases with Prefix Wildcard", func() {
+		t := d.T()
+
+		// create the databases
+		createDatabases(t)
+
+		deleted := databases[:2]
+		notDeleted := databases[2:]
+		// drop databases with a prefix wildcard
+		numDeleted, err := d.server.DropMultipleSensorDatabases("bingbong", true, false)
+		require.NoError(t, err, "dropping databases with prefix wildcard should not produce an error")
+		require.Equal(t, 2, numDeleted, "should delete exactly 2 databases") // should match the first two databases created
+
+		for _, dbName := range deleted {
+			d.checkDatabaseDeletion(dbName)
+		}
+
+		for _, dbName := range notDeleted {
+			d.checkDatabaseNonDeletion(dbName)
+		}
+	})
+
+	d.Run("Drop Databases with Suffix Wildcard", func() {
+		t := d.T()
+		// create the databases
+		createDatabases(t)
+
+		deleted := []string{databases[0], databases[2]}
+		notDeleted := []string{databases[1], databases[3]}
+		// drop databases with a suffix wildcard
+		numDeleted, err := d.server.DropMultipleSensorDatabases("bingbong", false, true)
+		require.NoError(t, err, "dropping databases with suffix wildcard should not produce an error")
+		require.Equal(t, 2, numDeleted, "should delete exactly 2 databases") // should match the first and third databases created
+
+		for _, dbName := range deleted {
+			d.checkDatabaseDeletion(dbName)
+		}
+
+		for _, dbName := range notDeleted {
+			d.checkDatabaseNonDeletion(dbName)
+		}
+	})
+
+	d.Run("Drop Databases with Both Wildcards", func() {
+		t := d.T()
+		// create the databases
+		createDatabases(t)
+
+		// drop databases with both wildcards
+		numDeleted, err := d.server.DropMultipleSensorDatabases("bingbong", true, true)
+		require.NoError(t, err, "dropping databases with both wildcards should not produce an error")
+		require.Equal(t, 4, numDeleted, "should delete all databases matching the wildcard pattern") // should match all databases created
+
+		for _, dbName := range databases {
+			d.checkDatabaseDeletion(dbName)
+		}
+	})
+
+	d.Run("Drop Databases with No Wildcards", func() {
+		t := d.T()
+		// create the databases
+		createDatabases(t)
+
+		// drop databases without wildcards
+		numDeleted, err := d.server.DropMultipleSensorDatabases("bingbong", false, false)
+		require.Error(t, err, "dropping databases without specifying a wildcard should produce an error")
+		require.Equal(t, 0, numDeleted, "no databases should be deleted if no wildcard is specified")
+		for _, dbName := range databases {
+			d.checkDatabaseNonDeletion(dbName)
+		}
+	})
+}
+
+func (d *DatabaseTestSuite) TestListImportDatabases() {
+	d.Run("List Databases", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, true)
+		require.NoError(t, err, "importing data should not produce an error")
+		_, err = cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB2", false, true)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		dbs, err := d.server.ListImportDatabases()
+		require.NoError(t, err, "listing databases should not produce an error")
+		require.Len(t, dbs, 2, "two databases should be listed")
+		dbString := database.GetFlatDatabaseList(dbs)
+
+		require.Containsf(t, dbString, "testDB", "testDB should be listed")
+		require.Contains(t, dbString, "testDB2", "testDB2 should be listed")
+	})
+
+	d.Run("No Databases", func() {
+		t := d.T()
+		dbs, err := d.server.ListImportDatabases()
+		require.NoError(t, err, "listing databases should not produce an error")
+		require.Nil(t, dbs, "databases should be nil")
+	})
+
+	d.Run("Missing Metadatabase", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		err = d.server.Conn.Exec(context.Background(), "DROP DATABASE IF EXISTS metadatabase")
+		require.NoError(t, err, "dropping metadatabase should not produce an error")
+
+		dbs, err := d.server.ListImportDatabases()
+		require.NoError(t, err, "listing databases with missing metadatabase should not produce an error")
+		require.Nil(t, dbs, "databases should be nil")
+	})
+}
+
+func (d *DatabaseTestSuite) TestGetRollingStatus() {
+	d.Run("Get Status of Rolling Database", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", true, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		status, err := database.GetRollingStatus(context.Background(), d.server.Conn, "testDB")
+		require.NoError(t, err, "getting status of rolling database should not produce an error")
+		fmt.Print("status: ", status)
+		require.True(t, status, "status of rolling database should be true")
+	})
+
+	d.Run("Get Status of Non-Rolling Database", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		status, err := database.GetRollingStatus(context.Background(), d.server.Conn, "testDB")
+		require.NoError(t, err, "getting status of non-rolling database should not produce an error")
+		require.False(t, status, "status of non-rolling database should be false")
+	})
+
+	d.Run("Get Status of Non-Existent Database", func() {
+		t := d.T()
+		status, err := database.GetRollingStatus(context.Background(), d.server.Conn, "testDB")
+		require.Error(t, err, "getting status of non-existent database should produce an error")
+		require.Equal(t, err, database.ErrDatabaseNotFound, "error should be database not found")
+		require.False(t, status, "status of non-existent database should be false")
+	})
+
+}
+
+func (d *DatabaseTestSuite) TestDatabaseExists() {
+	d.Run("Database Exists", func() {
+		t := d.T()
+		_, err := cmd.RunImportCmd(time.Now(), d.cfg, afero.NewOsFs(), "../test_data/valid_tsv", "testDB", false, false)
+		require.NoError(t, err, "importing data should not produce an error")
+
+		exists, err := database.DatabaseExists(d.server.Conn, context.Background(), "testDB")
+		require.NoError(t, err, "checking if database exists should not produce an error")
+		require.True(t, exists, "database should exist")
+	})
+
+	d.Run("Database Does Not Exist", func() {
+		t := d.T()
+		exists, err := database.DatabaseExists(d.server.Conn, context.Background(), "testDB")
+		require.NoError(t, err, "checking if database exists should not produce an error")
+		require.False(t, exists, "database should not exist")
+	})
+}

database/threat_intel.go

@@ -0,0 +1,451 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/logger"
+	"activecm/rita/util"
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"golang.org/x/time/rate"
+)
+
+// threatIntelFeed represents a threat intel feed source from config
+type threatIntelFeed struct {
+	LastModified time.Time
+	Online       bool
+	Existing     bool
+}
+
+// threatIntelFeedRecord represents a record in the threat_intel_feeds table
+type threatIntelFeedRecord struct {
+	Hash               util.FixedString `ch:"hash"`
+	Path               string           `ch:"path"`
+	Online             bool             `ch:"online"`
+	LastModifiedOnDisk time.Time        `ch:"last_modified_on_disk"` // time the custom feed file was last modified on the file system
+	LastModified       time.Time        `ch:"last_modified"`         // used for troubleshooting/seeing the last time it was updated in DB
+}
+
+// threatIntelFeedEntry represents a record in the threat_intel table
+type threatIntelFeedEntry struct {
+	Hash util.FixedString `ch:"hash"`
+	IP   netip.Addr       `ch:"ip"`
+	FQDN string           `ch:"fqdn"`
+}
+
+// createThreatIntelTables creates the threat intel tables in the metadatabase
+func (server *ServerConn) createThreatIntelTables() error {
+
+	// create table to store threat intel entries
+	err := server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.threat_intel (
+		hash FixedString(16),
+		ip IPv6,
+		fqdn String,
+	) ENGINE = MergeTree()
+	PRIMARY KEY (hash, fqdn, ip)
+	`)
+	if err != nil {
+		return err
+	}
+
+	// create table to store threat intel feeds and their last modified date
+	err = server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.threat_intel_feeds(
+		hash FixedString(16),
+		path String,
+		online Bool,
+		last_modified_on_disk DateTime('UTC'),
+		last_modified DateTime('UTC'),
+	) ENGINE = ReplacingMergeTree(last_modified)
+	ORDER BY (hash, path)
+	`)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// syncThreatIntelFeedsFromConfig updates the threat intel feeds in the metadatabase based on the config
+func (server *ServerConn) syncThreatIntelFeedsFromConfig(afs afero.Fs, cfg *config.Config) error {
+	logger := logger.GetLogger()
+
+	// get the list of threat intel feeds from the config
+	feeds, err := getThreatIntelFeeds(afs, cfg)
+	if err != nil {
+		return err
+	}
+
+	// get list of all feeds from the metadatabase
+	rows, err := server.Conn.Query(server.ctx, `
+		SELECT hash, path, online, most_recent_last_modified AS last_modified, last_modified_on_disk FROM (
+			SELECT  hash, path, online, max(last_modified) AS most_recent_last_modified, argMax(last_modified_on_disk, last_modified) AS last_modified_on_disk 
+			FROM metadatabase.threat_intel_feeds
+			GROUP BY hash, path, online
+		)
+	`)
+	if err != nil {
+		return err
+	}
+
+	// create a rate limiter to control the rate of writing to the database
+	limiter := rate.NewLimiter(5, 5)
+
+	// create a channel to write feed entries to the database
+	writer := NewBulkWriter(server, cfg, 1, "metadatabase", "threat_intel", "INSERT INTO metadatabase.threat_intel", limiter, false)
+	writer.Start(0)
+
+	// iterate over each existing feed in the database
+	for rows.Next() {
+
+		var entry threatIntelFeedRecord
+
+		err = rows.ScanStruct(&entry)
+		if err != nil {
+			return err
+		}
+
+		// check if feed was removed from the config
+		feedRemovedFromConfig := false
+		if res, ok := feeds[entry.Path]; !ok {
+			feedRemovedFromConfig = true
+		} else {
+			// mark feed as existing (record exists in the database)
+			res.Existing = true
+			feeds[entry.Path] = res
+		}
+
+		var feed io.ReadCloser
+
+		// process the feed as needed based on its status
+		switch {
+		// if feed was removed from the config, remove it from the database
+		case feedRemovedFromConfig:
+			logger.Warn().Str("feed_path", entry.Path).Msg("[THREAT INTEL] Removing threat intel feed because it is no longer listed in the config")
+			// remove feed from database
+			if err = server.removeFeed(entry.Hash); err != nil {
+				return err
+			}
+			// skip to next feed
+			continue
+
+		// if feed has no last modified date on disk, update as online feed
+		case entry.Online:
+			logger.Info().Str("feed_url", entry.Path).Msg("[THREAT INTEL] Updating online feed...")
+
+			// download the feed
+			feed, err = getOnlineFeed(server.GetContext(), entry.Path)
+			if err != nil {
+				return err
+			}
+
+		// if feed has has an oudated last modified date, update as custom feed
+		case entry.LastModifiedOnDisk != feeds[entry.Path].LastModified:
+			logger.Info().Str("feed_path", entry.Path).Msg("[THREAT INTEL] Updating custom feed because it has been modified...")
+			// open the feed file
+			feed, err = getCustomFeed(entry.Path)
+			if err != nil {
+				return err
+			}
+
+		// feed is up to date, skip ahead to next feed
+		default:
+			continue
+
+		}
+
+		// update the feed record in the database
+		if err = server.updateFeed(entry, feeds[entry.Path].LastModified, feed, writer.WriteChannel); err != nil {
+			return err
+		}
+
+	}
+	// iterate over each feed in the config that was not in the database
+	for path, entry := range feeds {
+		if !entry.Existing {
+			var feed io.ReadCloser
+			if entry.Online {
+				// download the feed
+				feed, err = getOnlineFeed(server.GetContext(), path)
+				if err != nil {
+					return err
+				}
+				logger.Info().Str("feed_url", path).Msg("[THREAT INTEL] Adding new online feed...")
+
+			} else {
+				// open the feed file
+				feed, err = getCustomFeed(path)
+				if err != nil {
+					return err
+				}
+				logger.Info().Str("feed_path", path).Msg("[THREAT INTEL] Adding new custom feed...")
+
+			}
+
+			// add the new feed to the database
+			if err = server.addNewFeed(path, entry, feed, writer.WriteChannel); err != nil {
+				return err
+			}
+		}
+	}
+	writer.Close()
+	return nil
+}
+
+// fs := afero.NewOsFs()
+// getThreatIntelFeeds parses the threat intel sources from the config file into a feed map
+func getThreatIntelFeeds(afs afero.Fs, cfg *config.Config) (map[string]threatIntelFeed, error) {
+	feeds := make(map[string]threatIntelFeed)
+	// add custom feed sources
+	if err := getCustomFeedsList(afs, feeds, cfg.ThreatIntel.CustomFeedsDirectory); err != nil {
+		return nil, err
+	}
+
+	// add online feed sources (with last modified time set to zero)
+	getOnlineFeedsList(feeds, cfg.ThreatIntel.OnlineFeeds)
+
+	return feeds, nil
+}
+
+// getCustomFeedsList populates the feeds map with the custom feed files contained in a specified directory
+// and their last modified times
+func getCustomFeedsList(afs afero.Fs, feeds map[string]threatIntelFeed, dirPath string) error {
+	logger := logger.GetLogger()
+
+	feedDir, err := util.ParseRelativePath(dirPath)
+	if err != nil {
+		return err
+	}
+
+	logger.Debug().Str("directory", feedDir).Msg("custom feed directory for threat intel")
+	// check if directory is valid
+	err = util.ValidateDirectory(afs, feedDir)
+	if err != nil {
+		// return nil if the directory doesn't exist or contains no files
+		if errors.Is(err, util.ErrDirDoesNotExist) || errors.Is(err, util.ErrDirIsEmpty) {
+			return nil
+		}
+		return err
+	}
+
+	// walk the directory and add each file to the feeds map
+	err = afero.Walk(afs, feedDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if filepath.Ext(path) == ".txt" {
+				feeds[path] = threatIntelFeed{
+					LastModified: info.ModTime().UTC().Truncate(time.Second),
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getOnlineFeedsList populates the feeds map with the passed in online feed sources (with last modified time set to zero)
+func getOnlineFeedsList(feeds map[string]threatIntelFeed, onlineFeedsList []string) {
+	for _, feed := range onlineFeedsList {
+		feeds[feed] = threatIntelFeed{
+			Online: true,
+		}
+	}
+}
+
+// getOnlineFeed gets the feed at the specified URL and returns an io.ReadCloser
+func getOnlineFeed(ctx context.Context, url string) (io.ReadCloser, error) {
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.Body, nil
+}
+
+// getCustomFeed opens the custom feed from the specified path and returns an io.ReadCloser
+func getCustomFeed(path string) (io.ReadCloser, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (server *ServerConn) updateFeed(entry threatIntelFeedRecord, lastModified time.Time, feed io.ReadCloser, writeChan chan Data) error {
+	// clear feed from database
+	if err := server.removeFeedEntries(entry.Hash); err != nil {
+		return err
+	}
+
+	// update feed record in database
+	// update last modified date to the last date the path was modified
+	entry.LastModifiedOnDisk = lastModified
+	if err := server.createFeedRecord(entry); err != nil {
+		return err
+	}
+
+	// upload the feed to the database
+	if err := parseFeedEntries(entry.Hash, feed, writeChan); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (server *ServerConn) addNewFeed(path string, entry threatIntelFeed, feed io.ReadCloser, writeChan chan Data) error {
+	// get hash of the feed path
+	hash, err := util.NewFixedStringHash(path)
+	if err != nil {
+		return err
+	}
+
+	// create a new feed record
+	record := threatIntelFeedRecord{
+		Hash:               hash,
+		Path:               path,
+		Online:             entry.Online,
+		LastModifiedOnDisk: entry.LastModified,
+	}
+
+	// create the feed record in the database
+	if err := server.createFeedRecord(record); err != nil {
+		return err
+	}
+
+	// upload the feed entries to the database
+	if err := parseFeedEntries(record.Hash, feed, writeChan); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (server *ServerConn) removeFeed(hash util.FixedString) error {
+	// remove feed record from threat_intel_feeds table
+	if err := server.removeFeedRecord(hash); err != nil {
+		return err
+	}
+
+	// remove feed entries from threat_intel table
+	if err := server.removeFeedEntries(hash); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// createFeedRecord adds a feed record to the metadatabase to track a threat intel feed
+func (server *ServerConn) createFeedRecord(record threatIntelFeedRecord) error {
+	record.LastModified = time.Now().UTC()
+
+	err := server.Conn.Exec(server.ctx, `
+		INSERT INTO metadatabase.threat_intel_feeds (
+			hash, path, online, last_modified_on_disk, last_modified
+		) VALUES (
+			unhex(?), ?, ?, ?, ?
+		)
+	`, record.Hash.Hex(), record.Path, record.Online, record.LastModifiedOnDisk, record.LastModified)
+	return err
+}
+
+// removeFeedRecord removes a threat intel feed record from the metadatabase collection for threat intel feeds
+func (server *ServerConn) removeFeedRecord(hash util.FixedString) error {
+	// set context parameters
+	ctx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{"hash": hash.Hex()}))
+
+	err := server.Conn.Exec(ctx, `
+		DELETE FROM metadatabase.threat_intel_feeds
+		WHERE hash = unhex({hash:String})
+	`)
+
+	return err
+}
+
+// parseFeedEntries parses a feed from an io.ReadCloser and sends valid entries on writeChan
+func parseFeedEntries(feedHash util.FixedString, feed io.ReadCloser, writeChan chan Data) error {
+	reader := bufio.NewReader(feed)
+
+	for {
+		line, readErr := reader.ReadString('\n')
+		// fmt.Println(line)
+
+		// if there is an error reading the line and its not the end of the file, return the error
+		if readErr != nil && readErr != io.EOF {
+			return readErr
+		}
+
+		// if this is the end of the file and the final line is empty, just break the loop
+		if len(line) < 1 && readErr == io.EOF {
+			break
+		}
+
+		// skip if line is comment based on  most common comment characters
+		if line[0] == '#' || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "<!--") {
+			continue
+		}
+
+		// remove leading/trailing spaces and newline characters
+		line = strings.TrimSpace(line)
+
+		feedEntry := &threatIntelFeedEntry{
+			Hash: feedHash,
+		}
+		// attempt to parse string as IP address
+		ip, err := netip.ParseAddr(line)
+		if err != nil {
+			// if it's not an IP, try parsing as fqdn
+			if util.ValidFQDN(line) {
+				// send fqdn to writer
+				feedEntry.FQDN = line
+				writeChan <- feedEntry
+			}
+		} else {
+			// send IP as IPv6 to writer
+			feedEntry.IP = ip
+			writeChan <- feedEntry
+		}
+
+		// if we have reached the end of the file, break the loop
+		if readErr == io.EOF {
+			break // End of file
+		}
+	}
+	feed.Close()
+
+	return nil
+}
+
+// removeFeedEntries removes entries associated with a threat intel feed from the metadatabase
+func (server *ServerConn) removeFeedEntries(hash util.FixedString) error {
+	// set context parameters
+	ctx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{"hash": hash.Hex()}))
+
+	err := server.Conn.Exec(ctx, `
+		DELETE FROM metadatabase.threat_intel
+		WHERE hash = unhex({hash:String})
+	`)
+
+	return err
+}

database/threat_intel_test.go

@@ -0,0 +1,122 @@
+package database
+
+import (
+	"activecm/rita/util"
+	"bufio"
+	"context"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseOnlineFeeds(t *testing.T) {
+	// TEST IP ONLINE FEED
+	t.Run("IP Online Feed", func(t *testing.T) {
+		// should be able to parse Feodo tracker
+		c := make(chan Data)
+		expectedTotal := 0
+		total := 0
+
+		// make a go routine to read from the channel and increment total
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range c {
+				total++
+			}
+		}()
+
+		// get expected total from last line of feed
+		feed, err := getOnlineFeed(context.Background(), "https://feodotracker.abuse.ch/downloads/ipblocklist.txt")
+		require.NoError(t, err, "getting online feed should not error")
+		reader := bufio.NewReader(feed)
+		for {
+			line, err := reader.ReadString('\n')
+			line = strings.TrimSpace(line)
+
+			if strings.HasPrefix(line, "# END") {
+				re := regexp.MustCompile(`\d+`)
+				// Find the first match
+				match := re.FindString(line)
+				require.NotEmpty(t, match, "match should not be empty")
+
+				// Convert the matched string to an integer
+				number, err := strconv.Atoi(match)
+				require.NoError(t, err, "converting string to int should not error")
+				expectedTotal = number
+			}
+			if err == io.EOF {
+				break // End of file
+			}
+		}
+		// make sure expected total is greater than zero
+		require.Greater(t, expectedTotal, 0, "expected total should be greater than zero")
+		feed.Close()
+
+		// read feed again
+		url := "https://feodotracker.abuse.ch/downloads/ipblocklist.txt"
+		feed, err = getOnlineFeed(context.Background(), url)
+		require.NoError(t, err, "getting online feed should not produce an error")
+
+		// get hash
+		hash, err := util.NewFixedStringHash(url)
+		require.NoError(t, err, "calculating hash should not produce an error")
+		require.NotEmpty(t, hash, "hash should not be empty")
+
+		// parse feed entries
+		err = parseFeedEntries(hash, feed, c)
+		require.NoError(t, err, "parsing feed entries should not produce an error")
+
+		// close channel and wait for go routine to finish
+		close(c)
+		wg.Wait()
+
+		// verify that feed matches expected total
+		require.EqualValues(t, expectedTotal, total, "total should match expected value")
+	})
+
+	// TEST DOMAIN ONLINE FEED
+	t.Run("Domain Online Feed", func(t *testing.T) {
+		// create a channel to mimic the writer which would receive the parsed data
+		d := make(chan Data)
+		total := 0
+
+		// make a go routine to read from the channel and increment total
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range d {
+				total++
+			}
+		}()
+
+		// get feed
+		url := "https://publicsuffix.org/list/public_suffix_list.dat"
+		feed, err := getOnlineFeed(context.Background(), url)
+		require.NoError(t, err, "getting online feed should not error")
+
+		// get hash
+		hash, err := util.NewFixedStringHash(url)
+		require.NoError(t, err, "calculating hash should not error")
+		require.NotEmpty(t, hash, "hash should not be empty")
+
+		// parse feed entries
+		err = parseFeedEntries(hash, feed, d)
+		require.NoError(t, err, "parsing feed entries should not error")
+
+		// close channel and wait for go routine to finish
+		close(d)
+		wg.Wait()
+
+		// make sure at least one fqdn was parsed
+		require.Greater(t, total, 0, "at least one fqdn should have been parsed")
+
+	})
+}

database/ttl.go

@@ -0,0 +1,175 @@
+package database
+
+import (
+	"fmt"
+	"strconv"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+)
+
+// Check the status of a table's TTL:
+// SELECT
+// 	delete_ttl_info_min,
+// 	delete_ttl_info_max
+// FROM system.parts
+// WHERE database='chickenstrip' and table = 'conn'
+
+var LogTableTTLs = []string{"conn", "http", "ssl", "dns", "pdns_raw"}
+var LogTableViewsHourTTLs = []string{"usni", "udns", "uconn", "mime_type_uris"}
+var LogTableViewsDayTTLs = []string{"pdns"}
+var AnalysisSnapshotHourTTLs = []string{"big_ol_histogram", "tls_proto", "http_proto", "exploded_dns", "rare_signatures", "port_info"}
+var AnalysisSnapshotAnalyzedAtTTLs = []string{"threat_mixtape"}
+var MetaDatabaseTTLs = []string{"historical_first_seen", "files"}
+var MetaDatabaseYearTTLS = []string{"imports"}
+
+func (db *DB) createLogTableTTLs() error {
+	if !db.Rolling {
+		return fmt.Errorf("cannot create TTLs on non-rolling database: %s", db.selected)
+	}
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+
+	err := db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.conn MODIFY TTL import_time + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.http MODIFY TTL import_time + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.ssl MODIFY TTL import_time + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.dns MODIFY TTL import_time + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.pdns_raw MODIFY TTL import_time + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	// tables populated by materialized views [ TTL on import_hour ]
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.usni MODIFY TTL import_hour + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.udns MODIFY TTL import_hour + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.uconn MODIFY TTL import_hour + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.pdns MODIFY TTL import_day + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.mime_type_uris MODIFY TTL import_hour + INTERVAL 26 HOURS`)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (db *DB) createSnapshotTableTTLs() error {
+	if !db.Rolling {
+		return fmt.Errorf("cannot create 'snapshot' TTLs on non-rolling database: %s", db.selected)
+	}
+	ctx := db.QueryParameters(clickhouse.Parameters{
+		"database": db.selected,
+	})
+
+	err := db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.big_ol_histogram MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.tls_proto MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.http_proto MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.exploded_dns MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.rare_signatures MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.port_info MODIFY TTL import_hour + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	err = db.Conn.Exec(ctx, `--sql
+		ALTER TABLE {database:Identifier}.threat_mixtape MODIFY TTL toDateTime(analyzed_at) + INTERVAL 2 WEEKS`)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (server *ServerConn) createMetaDatabaseTTLs(monthsToKeepHistoricalFirstSeen int) error {
+	ctx := clickhouse.Context(server.ctx, clickhouse.WithParameters(clickhouse.Parameters{
+		"days": strconv.Itoa(monthsToKeepHistoricalFirstSeen * 30),
+	}))
+
+	err := server.Conn.Exec(ctx, `--sql
+		ALTER TABLE metadatabase.historical_first_seen MODIFY TTL last_seen + toIntervalDay({days:Int32})`)
+	if err != nil {
+		return err
+	}
+
+	err = server.Conn.Exec(ctx, `--sql
+		ALTER TABLE metadatabase.files MODIFY TTL ts + INTERVAL 180 DAYS DELETE WHERE rolling = true`)
+	if err != nil {
+		return err
+	}
+
+	// DO NOT SET TTL ON ended_at, WILL BREAK
+	err = server.Conn.Exec(ctx, `--sql
+		ALTER TABLE metadatabase.imports MODIFY TTL toDateTime(started_at) + INTERVAL 1 YEAR`)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

database/valid_mime_types.go

@@ -0,0 +1,135 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/util"
+	"encoding/csv"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/spf13/afero"
+	"golang.org/x/time/rate"
+)
+
+type ValidMIMEType struct {
+	MIMEType  string `ch:"mime_type"`
+	Extension string `ch:"extension"`
+}
+
+// createValidMIMETypeTable creates a table that stores MIME types beginning with "text" (text/css) and their associated extensions
+func (server *ServerConn) createValidMIMETypeTable() error {
+	err := server.Conn.Exec(server.ctx, `
+		DROP TABLE IF EXISTS metadatabase.valid_mime_types
+	`)
+	if err != nil {
+		return err
+	}
+	err = server.Conn.Exec(server.ctx, `
+		CREATE TABLE IF NOT EXISTS metadatabase.valid_mime_types (
+			mime_type String,
+			extension String
+		) ENGINE = MergeTree()
+		PRIMARY KEY (mime_type)
+	`)
+
+	return err
+
+}
+
+func (server *ServerConn) importValidMIMETypes(afs afero.Fs, cfg *config.Config) error {
+	// create a rate limiter to control the rate of writing to the database
+	limiter := rate.NewLimiter(5, 5)
+
+	// create a channel to write mime type entries to the database
+	writer := NewBulkWriter(server, cfg, 1, "metadatabase", "valid_mime_types", "INSERT INTO metadatabase.valid_mime_types", limiter, false)
+	writer.Start(0)
+
+	extFile, err := util.ParseRelativePath(cfg.HTTPExtensionsFilePath)
+	if err != nil {
+		return err
+	}
+	if err := readValidTextMIMETypeFile(extFile, writer.WriteChannel); err != nil {
+		return err
+	}
+
+	writer.Close()
+
+	return nil
+}
+
+func readValidTextMIMETypeFile(filePath string, writeChan chan Data) error {
+	file, err := os.Open(filePath)
+
+	// Checks for the error
+	if err != nil {
+
+		log.Fatal("Error while reading the file", err)
+		return err
+	}
+
+	// Closes the file
+	defer file.Close()
+
+	// The csv.NewReader() function is called in
+	// which the object os.File passed as its parameter
+	// and this creates a new csv.Reader that reads
+	// from the file
+	reader := csv.NewReader(file)
+
+	// ReadAll reads all the records from the CSV file
+	// and Returns them as slice of slices of string
+	// and an error if any
+	records, err := reader.ReadAll()
+
+	// Checks for the error
+	if err != nil {
+		fmt.Println("Error reading records")
+		return err
+	}
+
+	// Loop to iterate through
+	// and print each of the string slice
+	for _, line := range records {
+		if len(line) < 4 {
+			return errors.New("valid MIME type CSV does not contain at least 4 columns")
+		}
+		mimeType := line[1]
+		if len(mimeType) < 1 {
+			continue
+		}
+		extension := line[2]
+
+		if extension == "none" {
+			extension = ""
+		}
+
+		// remove dots from extension names
+		extension = strings.ReplaceAll(extension, ".", "")
+
+		// if extensions is a list, create a row for each one
+		extensions := strings.Split(extension, ",")
+		if len(extensions) > 1 {
+			for _, ext := range extensions {
+
+				ext = strings.TrimSpace(ext)
+				if len(ext) > 0 {
+					entry := &ValidMIMEType{
+						MIMEType:  mimeType,
+						Extension: ext,
+					}
+					writeChan <- entry
+				}
+			}
+		} else {
+			entry := &ValidMIMEType{
+				MIMEType:  mimeType,
+				Extension: extension,
+			}
+			writeChan <- entry
+		}
+	}
+	return nil
+}

database/valid_mime_types_test.go

@@ -0,0 +1,192 @@
+package database
+
+import (
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// go test -v ./database -run TestReadValidTextMIMETypeFile
+func TestReadValidTextMIMETypeFile(t *testing.T) {
+
+	tests := []struct {
+		name              string
+		dataCSV           string
+		writeChan         chan Data
+		expectedMIMETypes []*ValidMIMEType
+		expectedTotal     int
+		expectedError     bool
+	}{
+
+		{
+			name: "Simple",
+			dataCSV: `a,b,c,d
+					  1,2,3,4`,
+			writeChan: make(chan Data),
+			expectedMIMETypes: []*ValidMIMEType{
+				{
+					MIMEType:  "b",
+					Extension: "c",
+				},
+				{
+					MIMEType:  "2",
+					Extension: "3",
+				},
+			},
+			expectedTotal: 2,
+			expectedError: false,
+		},
+
+		{
+			name: "Valid MIME type file",
+			dataCSV: `css,text/css,.css,[RFC2318]
+					  csv,text/csv,.csv,[RFC4180][RFC7111]`,
+			writeChan: make(chan Data),
+			expectedMIMETypes: []*ValidMIMEType{
+				{
+					MIMEType:  "text/css",
+					Extension: "css",
+				},
+				{
+					MIMEType:  "text/csv",
+					Extension: "csv",
+				},
+			},
+			expectedTotal: 2,
+			expectedError: false,
+		},
+		{
+			name: "Test for Multiple Extensions",
+			dataCSV: `plain,text/plain,".txt, .ps",[RFC2046][RFC3676][RFC5147]
+					  markdown,text/markdown,".md, .markdown",[RFC7763]
+					  javascript,text/javascript,".js, .mjs., es, .mjs",[RFC9239]
+					  html,text/html,none,[RFC21221]`,
+			writeChan: make(chan Data),
+			expectedMIMETypes: []*ValidMIMEType{
+				{
+					MIMEType:  "text/plain",
+					Extension: "txt",
+				},
+				{
+					MIMEType:  "text/plain",
+					Extension: "ps",
+				},
+				{
+					MIMEType:  "text/markdown",
+					Extension: "md",
+				},
+				{
+					MIMEType:  "text/markdown",
+					Extension: "markdown",
+				},
+				{
+					MIMEType:  "text/javascript",
+					Extension: "js",
+				},
+				{
+					MIMEType:  "text/javascript",
+					Extension: "mjs",
+				},
+				{
+					MIMEType:  "text/javascript",
+					Extension: "es",
+				},
+				{
+					MIMEType:  "text/javascript",
+					Extension: "mjs",
+				},
+				{
+					MIMEType:  "text/html",
+					Extension: "",
+				},
+			},
+			expectedTotal: 9,
+			expectedError: false,
+		},
+		{
+			name: "Valid File with an Entry with Empty MIME Type",
+			dataCSV: `css,text/css,.css,[RFC2318]
+					  csv,text/csv,.csv,[RFC4180][RFC7111]
+					  ,,.txt,[RFC2046][RFC3676][RFC5147]`,
+			writeChan: make(chan Data),
+			expectedMIMETypes: []*ValidMIMEType{
+				{
+					MIMEType:  "text/css",
+					Extension: "css",
+				},
+				{
+					MIMEType:  "text/csv",
+					Extension: "csv",
+				},
+			},
+			expectedTotal: 2,
+			expectedError: false,
+		},
+		{
+			name: "Invalid MIME type file - less than 4 columns",
+			dataCSV: `a,b,c
+					  1,2,3`,
+			writeChan:         make(chan Data),
+			expectedMIMETypes: []*ValidMIMEType{},
+			expectedTotal:     0,
+			expectedError:     true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// require := require.New(t)
+
+			// create a temporary file to read from
+			tmpFile, err := os.CreateTemp("", "test-mime-types-*.csv")
+			require.NoError(t, err, "reading from temporary file should not produce an error")
+
+			// clean up after the test
+			defer os.Remove(tmpFile.Name())
+
+			// write the CSV to temporary file
+			_, err = tmpFile.Write([]byte(test.dataCSV))
+			require.NoError(t, err, "writing to temporary file should not produce an error")
+			require.NotEmpty(t, tmpFile, "temporary file should not be empty")
+
+			// close temporary file
+			err = tmpFile.Close()
+			require.NoError(t, err, "closing temporary file should not produce an error")
+
+			wg := sync.WaitGroup{}
+			wg.Add(1)
+			total := 0
+			go func() {
+				defer wg.Done()
+				for entry := range test.writeChan {
+					require := require.New(t)
+					// cast entry to ValidMIMEType
+					validMIMEType, ok := entry.(*ValidMIMEType)
+					require.True(ok, "entry should be of type *ValidMIMEType")
+
+					// check if the MIME type is in the list of expected MIME types
+					require.Contains(test.expectedMIMETypes, validMIMEType, "MIME type should be in the list of expected MIME types")
+
+					total++
+				}
+			}()
+
+			// run the function
+			err = readValidTextMIMETypeFile(tmpFile.Name(), test.writeChan)
+
+			// check if the error is as expected
+			require.Equal(t, test.expectedError, err != nil, "error should match expected value")
+
+			// close the write channel and wait for the goroutine to finish
+			close(test.writeChan)
+			wg.Wait()
+
+			// check if the total number of MIME types is as expected
+			require.Equal(t, test.expectedTotal, total, "total number of MIME types should match expected value")
+
+		})
+	}
+
+}

database/writer.go

@@ -0,0 +1,243 @@
+package database
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/logger"
+	"context"
+	"sync"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	driver "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/time/rate"
+)
+
+type (
+	Data any
+
+	// Interface to allow creating a BulkWriter from a DB or a serverConn
+	Database interface {
+		getConn() driver.Conn
+		GetContext() context.Context
+		QueryParameters(clickhouse.Parameters) context.Context
+	}
+
+	BulkWriter struct {
+		db           Database
+		conf         *config.Config
+		WriteChannel chan Data
+		ProgChannel  chan int
+		WriteWg      *errgroup.Group // wait for writing to finish
+		writerName   string          // used in error reporting
+		batchSize    int
+		query        string
+		limiter      *rate.Limiter
+		withProgress bool
+		database     string
+		closed       bool
+		ctx          context.Context
+		numWorkers   int
+		batches      []int
+		mu           sync.Mutex
+		cond         *sync.Cond
+	}
+)
+
+// NewBulkWriter creates a new writer object to write output data to collections
+func NewBulkWriter(db Database, conf *config.Config, numWorkers int, database string, writerName string, query string, limiter *rate.Limiter, withProgress bool) *BulkWriter {
+
+	analysisErrGroup, ctx := errgroup.WithContext(context.Background())
+	writer := &BulkWriter{
+		db:           db,
+		conf:         conf,
+		database:     database,
+		WriteChannel: make(chan Data),
+		ProgChannel:  make(chan int),
+		WriteWg:      analysisErrGroup,
+		writerName:   writerName,
+		batchSize:    conf.BatchSize,
+		query:        query,
+		limiter:      limiter,
+		withProgress: withProgress,
+		numWorkers:   numWorkers,
+		ctx:          ctx,
+		batches:      make([]int, numWorkers), // keeps track of the batch count for each worker
+	}
+	writer.cond = sync.NewCond(&writer.mu)
+	return writer
+}
+
+// shouldReadData returns whether or not the thread with the passed in ID should read data from the write channel
+func (w *BulkWriter) shouldReadData(id int, empty bool) bool {
+	if w.numWorkers == 1 {
+		return true
+	}
+
+	var numInProgress int
+	for i, b := range w.batches {
+		if i != id {
+			// batch is in progress if it has at least 1 item, but less than the batch size
+			if b > 0 && b < w.batchSize {
+				numInProgress++
+			}
+		}
+	}
+	// we don't want a worker that's not currently in progress to read the rest of the items from the channel after it's closed
+	// because then the leftover data will get distributed between all of the workers, making 5 or so tiny batches, which is really bad
+	if w.closed {
+		// allow any worker to pass through the cond wait if the channel is empty
+		if empty {
+			return true
+		}
+		// if the channel isn't empty yet, allow any in progress workers to keep going, or a new one if none are processing
+		return w.batches[id] > 0 || numInProgress == 0
+	}
+
+	// a worker should start reading if there are no other workers currently reading in data
+	// or keep reading if it's already in progress
+	return numInProgress == 0 || w.batches[id] > 0
+}
+
+// Close waits for the write threads to finish
+func (w *BulkWriter) Close() {
+	logger := logger.GetLogger()
+	// tell workers that no more data will be sent on this channel
+	close(w.WriteChannel)
+	// mark the channel as closed
+	w.closed = true
+	// notify workers that the channel is closed
+	w.cond.Broadcast()
+	// wait for the errgroup
+	if err := w.WriteWg.Wait(); err != nil {
+		logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "close_writer").Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+	}
+
+	close(w.ProgChannel)
+}
+
+// Start kicks off a new write thread
+func (w *BulkWriter) Start(id int) {
+
+	w.WriteWg.Go(func() error {
+		logger := logger.GetLogger()
+
+		conn := w.db.getConn()
+
+		chCtx := w.db.QueryParameters(clickhouse.Parameters{
+			"database": w.database,
+		})
+
+		batchCount := 0
+
+		var items []Data
+
+		// loop over input channel
+		for {
+
+			w.mu.Lock()
+			// check to see if this thread should take in data
+			for !w.shouldReadData(id, len(w.WriteChannel) == 0) {
+				// wait for other threads to process data if it isn't supposed to read in data yet
+				w.cond.Wait()
+			}
+
+			// check if any other workers errored out and made the context finish
+			select {
+			case <-w.ctx.Done():
+				return w.ctx.Err()
+			default:
+			}
+
+			// attempt to read data from the channel
+			change, ok := <-w.WriteChannel
+
+			// if the channel is closed, unlock the mutex and break out of the loop
+			if !ok {
+				w.mu.Unlock()
+				break // Exit if the channel is closed
+			}
+			// increment batch count
+			w.batches[id]++
+			batchCount++
+			// unlock mutex
+			w.mu.Unlock()
+
+			// add this data to the batch buffer
+			items = append(items, change)
+
+			// if batch size limit reached, write out batch of records
+			if batchCount >= w.batchSize {
+				// alert other workers that this worker is sending the batch so that
+				// a free worker can be allowed to start making a new batch
+				w.cond.Broadcast()
+
+				// initialize batch
+				batch, err := conn.PrepareBatch(chCtx, w.query)
+				if err != nil {
+					logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "prepare").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+				}
+
+				// add each item in batch to this batch
+				for _, item := range items {
+					err := batch.AppendStruct(item)
+					if err != nil {
+						logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "append").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+					}
+				}
+
+				// wait for the rate limiter so that not too many batches are inserted at a time
+				// ClickHouse recommends to send 1 batch per second, but it appears to work just fine for 5 batches per second
+				if err := w.limiter.Wait(w.db.GetContext()); err != nil {
+					logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "limiter").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+				}
+
+				// send batch
+				err = batch.Send()
+				if err != nil {
+					logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "send").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+				}
+
+				// if progress updates are enabled, send the number of records
+				// this batch handled on the progress channel
+				if w.withProgress {
+					w.ProgChannel <- batchCount
+				}
+
+				// update worker state batch count and alert other workers that this
+				// worker is empty
+				w.mu.Lock()
+				w.batches[id] = 0
+				w.cond.Broadcast()
+				w.mu.Unlock()
+				// reset count and items slice
+				batchCount = 0
+				items = nil
+			}
+		}
+
+		// handle batch when number of items is less than the batch size
+		if batchCount > 0 {
+			batch, err := conn.PrepareBatch(chCtx, w.query)
+			if err != nil {
+				logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "final_prepare").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+			}
+
+			for _, item := range items {
+				err := batch.AppendStruct(item)
+				if err != nil {
+					logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "final_append").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+				}
+			}
+
+			err = batch.Send()
+			if err != nil {
+				logger.Fatal().Err(err).Str("database", w.writerName).Str("stage", "final_send").Int("batch_size", w.batches[id]).Msg("Encountered an unrecoverable issue when trying to write to the database, exiting")
+			}
+
+			if w.withProgress {
+				w.ProgChannel <- batchCount
+			}
+		}
+		return nil
+	})
+}

default_config.hjson

@@ -0,0 +1,127 @@
+{
+    update_check_enabled: true,
+    log_level: 1,
+    logging_enabled: true,
+    threat_intel: {
+        // Configuration for custom threat intel feeds
+        // Allowed format for the contents of both online feeds and custom file feeds is one IP or domain per line
+        // Online feeds must be valid URLs
+        online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+        // MODIFY THE MOUNT DIRECTORY IN DOCKER COMPOSE, this should rarely need to be changed
+        custom_feeds_directory: "/etc/rita/threat_intel_feeds"
+    },
+    filtering: {
+        # These are filters that affect the import of connection logs. They
+        # currently do not apply to dns logs.
+        # A good reference for networks you may wish to consider is RFC 5735.
+        # https://tools.ietf.org/html/rfc5735#section-4
+
+        // internal_subnets identifies the internal network, which will result
+        // in any internal to internal and external to external connections being
+        // filtered out at import time. Reasonable defaults are provided below,
+        // but need to be manually verified before enabling. 
+        internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"], # Private-Use Networks  RFC 1918 and ULA prefix
+       
+        // always_included_subnets overrides the never_included_* and internal_subnets section,
+        // making sure that any connection records containing addresses from these arrays are kept and not filtered
+        // Note: the IP address of a proxy must be included here if the proxy is internal
+        always_included_subnets: [], // array of CIDRs
+        always_included_domains: [], // array of FQDNs
+
+        // connections involving ranges entered into never_included_subnets are filtered out at import time
+        never_included_subnets: [], // array of CIDRs
+        never_included_domains: [], // array of FQDNs
+        filter_external_to_internal: true // ignores any entries where communication is occurring from an external host to an internal host
+    },
+    scoring: {
+        beacon: {
+            // The default minimum number of unique connections used for beacons analysis.
+            // Any two hosts connecting fewer than this number will not be analyzed. You can 
+            // safely increase this value to improve performance if you are not concerned
+            //  about slow beacons.
+            unique_connection_threshold: 4, // min number of unique connections to qualify as beacon
+            
+            // The score is currently comprised of a weighted average of 4 subscores.
+            // While we recommend the default setting of 0.25 for each weight, 
+            // these weights can be altered here according to your needs. 
+            // The sum of all the floating point weights must be equal to 1.
+            timestamp_score_weight: 0.25,
+            datasize_score_weight: 0.25,
+            duration_score_weight: 0.25,
+            histogram_score_weight: 0.25,
+            // The number of hours seen in a connection graph representation of a beacon must
+            // be greater than this threshold for an overall duration score to be calculated.
+            // Default value: 6
+            duration_min_hours_seen: 6,
+            // This is the minimum number of hours seen in a connection graph representation
+            // of a beacon for the consistency subscore of duration to score at 100%
+            // Default value: 12 (half the day)
+            duration_consistency_ideal_hours_seen: 12,
+            // The histogram score has a subscore that attempts to detect multiple 
+            // flat sections in a connection graph representation of a beacon. The 
+            // variable below controls the bucket size for grouping connections.
+            // This is expressed as a percentage of the largest connection count. For example, 
+            // if the max connection count is 400 and this variable is set to 0.05 (5%), 
+            // the bucket size will be 20 (400*0.05=20). As you make this variable 
+            // larger, the algorithm becomes more forgiving to variation. 
+            // Default value 0.05
+            histogram_mode_sensitivity: 0.05,
+            // This is the number of buckets that can be considered outliers and dropped
+            // from the calculation.
+            // Default value: 1
+            histogram_bimodal_outlier_removal: 1,
+            // This is the minimum number of hours seen in a connection graph representation
+            // of a beacon before the bimodal subscore score is used.
+            // Default value: 11 (sets the minimum coverage to just below half of the day)
+            histogram_bimodal_min_hours_seen: 11,
+            score_thresholds: {
+                // beacon score
+                base: 50,
+                low: 70,
+                medium: 90,
+                high: 100
+            }
+        },
+        long_connection_minimum_duration: 3600,
+        long_connection_score_thresholds: {
+            // duration, in seconds
+            base: 3600, // 1 hour
+            low: 14400, // 4 hours
+            medium: 28800, // 8 hours
+            high: 43200 // 12 hours
+        },
+        c2_subdomain_threshold: 100,
+        c2_score_thresholds: {
+            // number of subdomains
+            base: 100,
+            low: 500,
+            medium: 800,
+            high: 1000
+        },
+        strobe_impact: {
+            category: "high" // any strobes will be placed in the high category
+        },
+        threat_intel_impact: {
+            category: "high" // any threat intel hits will be placed in the high category
+        }
+    },
+    modifiers: {
+        threat_intel_score_increase: 0.15, // score +15% if data size >= 25 MB
+        threat_intel_datasize_threshold: 25000000, // 25MB (as bytes)
+        prevalence_score_increase: 0.15, // score +15% if prevalence <= 2%
+        prevalence_increase_threshold: 0.02,
+        prevalence_score_decrease: 0.15, // score -15% if prevalence >= 50%
+        prevalence_decrease_threshold: 0.5, // must be greater than the increase threshold
+        first_seen_score_increase: 0.15, // score +15% if first seen <= 7 days ago
+        first_seen_increase_threshold: 7,
+        first_seen_score_decrease: 0.15, // score -15% if first seen >= 30 days ago
+        first_seen_decrease_threshold: 30, // must be greater than the increase threshold
+        missing_host_count_score_increase: 0.1, // +10% score for missing host header
+        rare_signature_score_increase: 0.15, // +15% score for connections with a rare signature
+        c2_over_dns_direct_conn_score_increase: 0.15, // +15% score for domains that were queried but had no direct connections
+        mime_type_mismatch_score_increase: 0.15 // +15% score for connections with mismatched MIME type/URI
+    },
+    http_extensions_file_path: "/http_extensions_list.csv", # path is relative to where it is in the container if run via docker
+    months_to_keep_historical_first_seen: 3,
+    batch_size: 100000
+}

deployment/config.xml

@@ -0,0 +1,8 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <mutations_sync>1</mutations_sync>
+            <optimize_aggregation_in_order>1</optimize_aggregation_in_order>
+        </default>
+    </profiles>
+</clickhouse>

deployment/http_extensions_list.csv

@@ -0,0 +1,93 @@
+1d-interleaved-parityfec,text/1d-interleaved-parityfec,none,[RFC6015]
+cache-manifest,text/cache-manifest,none,[W3C][Robin_Berjon]
+calendar,text/calendar,".ics, .ifb, .iCal, .iFBf",[RFC5545]
+cql,text/cql,none,[HL7][Bryn_Rhodes]
+cql-expression,text/cql-expression,none,[HL7][Bryn_Rhodes]
+cql-identifier,text/cql-identifier,none,[HL7][Bryn_Rhodes]
+css,text/css,.css,[RFC2318]
+csv,text/csv,.csv,[RFC4180][RFC7111]
+csv-schema,text/csv-schema,none,[National_Archives_UK][David_Underdown]
+directory - DEPRECATED by RFC6350,text/directory,none,[RFC2425][RFC6350]
+dns,text/dns,".soa, .zone",[RFC4027]
+ecmascript (OBSOLETED in favor of text/javascript),text/ecmascript,none,[RFC9239]
+encaprtp,text/encaprtp,none,[RFC6849]
+enriched,text/enriched,none,[RFC1896]
+example,text/example,none,[RFC4735]
+fhirpath,text/fhirpath,none,[HL7][Bryn_Rhodes]
+flexfec,text/flexfec,none,[RFC8627]
+fwdred,text/fwdred,none,[RFC6354]
+gff3,text/gff3,none,[Sequence_Ontology]
+grammar-ref-list,text/grammar-ref-list,none,[RFC6787]
+hl7v2,text/hl7v2,none,[HL7][Marc_Duteau]
+html,text/html,".html, .htm",[W3C][Robin_Berjon]
+javascript,text/javascript,".js, .mjs., es, .mjs",[RFC9239]
+jcr-cnd,text/jcr-cnd,none,[Peeter_Piegaze]
+markdown,text/markdown,".md, .markdown",[RFC7763]
+mizar,text/mizar,none,[Jesse_Alama]
+n3,text/n3,none,[W3C][Eric_Prudhommeaux]
+parameters,text/parameters,none,[RFC7826]
+parityfec,text/parityfec,none,[RFC3009]
+plain,text/plain,".txt, .ps",[RFC2046][RFC3676][RFC5147]
+provenance-notation,text/provenance-notation,none,[W3C][Ivan_Herman]
+prs.fallenstein.rst,text/prs.fallenstein.rst,none,[Benja_Fallenstein]
+prs.lines.tag,text/prs.lines.tag,none,[John_Lines]
+prs.prop.logic,text/prs.prop.logic,none,[Hans-Dieter_A._Hiep]
+prs.texi,text/prs.texi,none,[Matin_Bavardi]
+raptorfec,text/raptorfec,none,[RFC6682]
+RED,text/RED,none,[RFC4102]
+rfc822-headers,text/rfc822-headers,none,[RFC6522]
+rtf,text/rtf,none,[Paul_Lindner]
+rtp-enc-aescm128,text/rtp-enc-aescm128,none,[_3GPP]
+rtploopback,text/rtploopback,none,[RFC6849]
+rtx,text/rtx,none,[RFC4588]
+SGML,text/SGML,none,[RFC1874]
+shaclc,text/shaclc,none,[W3C_SHACL_Community_Group][Vladimir_Alexiev]
+shex,text/shex,none,[W3C][Eric_Prudhommeaux]
+spdx,text/spdx,none,[Linux_Foundation][Rose_Judge]
+strings,text/strings,none,[IEEE-ISTO-PWG-PPP]
+t140,text/t140,none,[RFC4103]
+tab-separated-values,text/tab-separated-values,none,[Paul_Lindner]
+troff,text/troff,none,[RFC4263]
+turtle,text/turtle,none,[W3C][Eric_Prudhommeaux]
+ulpfec,text/ulpfec,none,[RFC5109]
+uri-list,text/uri-list,".uris,.uri",[RFC2483]
+vcard,text/vcard,".ifb,.ics,.vcf .vcard",[RFC6350]
+vnd.a,text/vnd.a,none,[Regis_Dehoux]
+vnd.abc,text/vnd.abc,none,[Steve_Allen]
+vnd.ascii-art,text/vnd.ascii-art,none,[Kim_Scarborough]
+vnd.curl,text/vnd.curl,none,[Robert_Byrnes]
+vnd.debian.copyright,text/vnd.debian.copyright,none,[Charles_Plessy]
+vnd.DMClientScript,text/vnd.DMClientScript,none,[Dan_Bradley]
+vnd.dvb.subtitle,text/vnd.dvb.subtitle,none,[Peter_Siebert][Michael_Lagally]
+vnd.esmertec.theme-descriptor,text/vnd.esmertec.theme-descriptor,none,[Stefan_Eilemann]
+vnd.exchangeable,text/vnd.exchangeable,none,[Martin_Cizek]
+vnd.familysearch.gedcom,text/vnd.familysearch.gedcom,none,[Gordon_Clarke]
+vnd.ficlab.flt,text/vnd.ficlab.flt,none,[Steve_Gilberd]
+vnd.fly,text/vnd.fly,none,[John-Mark_Gurney]
+vnd.fmi.flexstor,text/vnd.fmi.flexstor,none,[Kari_E._Hurtta]
+vnd.gml,text/vnd.gml,none,[Mi_Tar]
+vnd.graphviz,text/vnd.graphviz,none,[John_Ellson]
+vnd.hans,text/vnd.hans,none,[Hill_Hanxv]
+vnd.hgl,text/vnd.hgl,none,[Heungsub_Lee]
+vnd.in3d.3dml,text/vnd.in3d.3dml,none,[Michael_Powers]
+vnd.in3d.spot,text/vnd.in3d.spot,none,[Michael_Powers]
+vnd.IPTC.NewsML,text/vnd.IPTC.NewsML,none,[IPTC]
+vnd.IPTC.NITF,text/vnd.IPTC.NITF,none,[IPTC]
+vnd.latex-z,text/vnd.latex-z,none,[Mikusiak_Lubos]
+vnd.motorola.reflex,text/vnd.motorola.reflex,none,[Mark_Patton]
+vnd.ms-mediapackage,text/vnd.ms-mediapackage,none,[Jan_Nelson]
+vnd.net2phone.commcenter.command,text/vnd.net2phone.commcenter.command,none,[Feiyu_Xie]
+vnd.radisys.msml-basic-layout,text/vnd.radisys.msml-basic-layout,none,[RFC5707]
+vnd.senx.warpscript,text/vnd.senx.warpscript,none,[Pierre_Papin]
+vnd.si.uricatalogue (OBSOLETED by request),text/vnd.si.uricatalogue,none,[Nicholas_Parks_Young]
+vnd.sun.j2me.app-descriptor,text/vnd.sun.j2me.app-descriptor,none,[Gary_Adams]
+vnd.sosi,text/vnd.sosi,none,[Petter_Reinholdtsen]
+vnd.trolltech.linguist,text/vnd.trolltech.linguist,none,[David_Lee_Lambert]
+vnd.wap.si,text/vnd.wap.si,none,[WAP-Forum]
+vnd.wap.sl,text/vnd.wap.sl,none,[WAP-Forum]
+vnd.wap.wml,text/vnd.wap.wml,none,[Peter_Stark]
+vnd.wap.wmlscript,text/vnd.wap.wmlscript,none,[Peter_Stark]
+vtt,text/vtt,none,[W3C][Silvia_Pfeiffer]
+wgsl,text/wgsl,none,[W3C][David_Neto]
+xml,text/xml," .xml, .ent, .dtd, .mod",[RFC7303]
+xml-external-parsed-entity,text/xml-external-parsed-entity," .xml, .ent, .dtd, .mod",[RFC7303]

deployment/logger-cron

@@ -0,0 +1 @@
+0 3 * * * /usr/bin/find /var/log -type f -daystart -mtime +90 -exec rm {} \;

deployment/syslog-ng.conf

@@ -0,0 +1,40 @@
+#############################################################################
+# Default syslog-ng.conf file which collects all local logs into a
+# single file called /var/log/messages tailored to container usage.
+@version: 4.5
+@include "scl.conf"
+
+source s_local {
+  internal();
+};
+
+source s_network_tcp {
+  syslog(transport(tcp) port(6601));
+};
+
+source s_network_udp {
+  syslog(transport(udp) port(5514));
+};
+
+destination d_local {
+  file("/config/logs/rita/rita_${YEAR}_${MONTH}_${DAY}.log" 
+    template("$ISODATE ${MESSAGE}\n") 
+    frac-digits(3)
+    create-dirs(yes)
+    template_escape(no)
+    log_fifo_size(1000)
+    dir-owner(root)
+    dir-group(root)
+    dir-perm(0750)
+    owner(root)
+    group(adm)
+    perm(0640)
+  );
+};
+
+log {
+  source(s_local);
+  source(s_network_tcp);
+  source(s_network_udp);
+  destination(d_local);
+};

deployment/timezone.xml

@@ -0,0 +1,3 @@
+<clickhouse>
+    <timezone>UTC</timezone>
+</clickhouse>

docker-compose.prod.yml

@@ -0,0 +1,69 @@
+volumes:
+  clickhouse_persistent:
+networks:
+  rita-network: {}
+services:
+  rita:
+    image: ghcr.io/activecm/rita-v2:latest
+    build: .
+    depends_on:
+      - clickhouse
+    volumes:
+      - ${CONFIG_FILE:-/etc/rita/config.hjson}:/config.hjson
+      - ${CONFIG_DIR:-/etc/rita}/http_extensions_list.csv:/http_extensions_list.csv
+      - /opt/rita/.env:/.env
+      # - ${LOGS:?"You must provide a directory for logs to be read from"}:/logs:ro
+    links:
+      - "clickhouse:db"
+      - "syslog-ng:syslogng"
+    environment:
+      - DB_ADDRESS=db:9000
+      - TERM=xterm-256color
+
+  syslog-ng:
+    image: lscr.io/linuxserver/syslog-ng:latest
+    container_name: rita-syslog-ng
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Etc/UTC
+    volumes:
+      # TODO: run the cron on the host, not in the container since this seems to cause issues
+      - ${CONFIG_DIR:-/etc/rita}/logger-cron:/etc/cron.d/logger-cron
+      - ${CONFIG_DIR:-/etc/rita}/syslog-ng.conf:/config/syslog-ng.conf
+      - ${APP_LOGS:-/var/log/rita}:/config/logs/rita
+    # ports:
+    #   - 514:5514/udp
+    #   - 601:6601/tcp
+    #   - 6514:6514/tcp
+    expose:
+      - 5514/udp
+      - 6601/tcp
+    restart: unless-stopped
+  clickhouse:
+    image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION?"Missing ClickHouse version"}
+    container_name: rita-clickhouse
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      start_period: 5s
+      retries: 30
+    restart: unless-stopped
+    expose:
+      - 9000
+    # ports:
+    #   - 127.0.0.1:8123:8123
+    #   - 127.0.0.1:9000:9000
+    volumes:
+      - type: bind
+        source: /etc/localtime
+        target: /etc/localtime
+        read_only: true
+      - clickhouse_persistent:/var/lib/clickhouse
+      # - /tmp/clickhouse_logs:/var/log/clickhouse-server
+      - ${CONFIG_DIR:-/etc/rita}/config.xml:/etc/clickhouse-server/users.d/custom_config.xml
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 131070
+        hard: 131070

docker-compose.yml

@@ -0,0 +1,65 @@
+volumes:
+  clickhouse_persistent:
+networks:
+  rita-network: {}
+services:
+  rita:
+    # image: 
+    build: .
+    depends_on:
+      - clickhouse
+    volumes:
+      - ${CONFIG_FILE:-/etc/rita/config.hjson}:/config.hjson
+      - ${CONFIG_DIR:-/etc/rita}/http_extensions_list.csv:/http_extensions_list.csv
+      - .env:/.env
+      # - ${LOGS:?"You must provide a directory for logs to be read from"}:/logs:ro
+    links:
+      - "clickhouse:db"
+      - "syslog-ng:syslogng"
+    environment:
+      - DB_ADDRESS=db:9000
+      - TERM=xterm-256color
+  syslog-ng:
+    image: lscr.io/linuxserver/syslog-ng:latest
+    container_name: syslog-ng
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Etc/UTC
+    volumes:
+      # TODO: run the cron on the host, not in the container since this seems to cause issues
+      - ${CONFIG_DIR:-/etc/rita}/logger-cron:/etc/cron.d/logger-cron
+      - ${CONFIG_DIR:-/etc/rita}/syslog-ng.conf:/config/syslog-ng.conf
+      - ${APP_LOGS:-/var/log/rita}:/config/logs/rita
+    ports:
+      - 514:5514/udp
+      # - 601:6601/tcp
+      # - 6514:6514/tcp
+    restart: unless-stopped
+  clickhouse:
+    image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION?"Missing ClickHouse version"}
+    container_name: clickhouse
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      start_period: 5s
+      retries: 30
+    restart: unless-stopped
+    ports:
+      - 127.0.0.1:8123:8123
+      - 127.0.0.1:9000:9000
+    volumes:
+      - type: bind
+        source: /etc/localtime
+        target: /etc/localtime
+        read_only: true
+      - clickhouse_persistent:/var/lib/clickhouse
+      # - /tmp/clickhouse_logs:/var/log/clickhouse-server
+      - ${CONFIG_DIR:-/etc/rita}/config.xml:/etc/clickhouse-server/users.d/custom_config.xml
+      - ${CONFIG_DIR:-/etc/rita}/timezone.xml:/etc/clickhouse-server/config.d/timezone.xml
+
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 131070
+        hard: 131070

docs/Configuration.md

@@ -0,0 +1,98 @@
+# Configuration
+
+
+RITA's behavior and performance can be fine-tuned using a configuration file. This configuration file allows you to adjust various settings, including scoring parameters that affect how different types of network activities are evaluated.
+
+## Location of the Configuration File
+The default configuration file is located [here](/default-config.hjson). 
+
+When RITA is installed, the config file is located at `/etc/rita/config.hjson`.
+
+You can specify a different configuration file location using the `-c` or `--config` flag when running RITA:
+
+```bash
+./rita -c /path/to/your/custom/config.conf <command> <flags>
+```
+
+## Fine-Tuning the Scoring
+The configuration file includes various parameters that control the scoring mechanism used by RITA. Adjusting these parameters can help you customize how different types of network threats are evaluated and scored.
+
+Below are some of the key sections and parameters you can adjust:
+
+### Scoring
+This section defines scoring parameters for each type of network threat. Please refer to the [default configuration file](/default-config.hjson) for a concise summary of each parameter and how it affects the detection and scoring of different threat types. This will provide a comprehensive overview of the available configuration options and how to customize them to meet your specific needs.  Below is an explanation of some common parameters that apply to multiple threat types.
+
+#### Score Thresholds
+The score_thresholds section defines the thresholds for categorizing the severity of network activities based on their scores. Each scoring category (e.g., beacon, long_connection) has its own thresholds for determining whether an activity falls into the base, low, medium, or high severity levels.
+
+Example:
+
+```yaml
+scoring: {
+    beacon: {
+        ...
+        score_thresholds: {
+            base: 50,
+            low: 70,
+            medium: 90,
+            high: 100
+        }
+    },
+    ...
+}
+```
+In this example, a beacon score of:
+
+Less than 50 is considered below the base threshold.
+Between 50 and 69 is considered low.
+Between 70 and 89 is considered medium.
+90 and above is considered high.
+
+#### Impact
+The impact sections in the scoring configuration determine the severity category for specific types of activities. For example, any activity flagged as a strobe or a threat intel hit can be placed in the high category regardless of their individual scores.
+
+Example:
+
+```yaml
+scoring: {
+    ...
+    strobe_impact: {
+        category: "high" // any strobes will be placed in the high category
+    },
+    threat_intel_impact: {
+        category: "high" // any threat intel hits will be placed in the high category
+    }
+}
+```
+In this example, any strobe or threat intel hit will be automatically categorized as high severity.
+
+*Note that the category cannot be set to "critical".*
+
+### Score Modification
+Scores for detected threats can be modified (increased or decreased) based on other behaviors detected. 
+
+The configuration for modifiers is in the `modifiers` object within the configuration file.
+
+Some modifiers only apply if a certain threshold is met, while other modifiers either apply or do not apply. For example:
+
+#### Modifier with thresholds:
+
+The Prevalence modifier increases the score of a threat by `prevalence_score_increase` (ex: `0.15` (+15%)) if the prevalence of the external host is less than or equal to the `prevalence_increase_threshold` (ex: `0.02` (2%)). 
+
+Inversely, the prevalence modifier also has a score decrease and a decrease threshold, where the threat score will decrease by `prevalence_score_decrease` if the prevalence is greater than or equal to the `prevalence_decrease_threshold`.
+
+#### Modifier without thresholds:
+
+The Missing Host Header modifier increases the threat score by `missing_host_count_score_increase` if the connection had no host header set.
+
+### Applying Configuration Changes
+After making changes to the configuration file, save the file and re-run RITA to apply the changes:
+
+```bash
+./rita -c /path/to/your/config/rita.conf <command> <flags>
+```
+To apply these changes immediately, the dataset will need to be destroyed and reimported. To do this, use the `--rebuild` flag.
+
+The changes will not *fully* propogate a rolling dataset until an import is made 24 hours after the config was changed.
+
+By adjusting these configuration parameters, you can fine-tune RITA's scoring to better match your network's characteristics and security policies.

docs/Development.md

@@ -0,0 +1,47 @@
+
+## Development
+There are two options for a development setup.
+
+The `.env` file is setup for the development build. The `.env.production` file is copied over to the RITA container during the image build process. When running RITA in Docker, the regular `.env` file is still loaded by Docker since the paths for the files to mount to are relative to the repository. A production install copies the `.env.production` file to `/opt/rita`.
+
+Note: Your usage of `sudo` in relation to these instructions may change depending on your system.
+
+#### Rapid Development
+For rapid development, RITA is not ran in Docker. The plain `docker-compose.yml` file exposes the database on the host! If you are running this in an environment where this is not ideal, please follow the instructions for Docker Development.
+
+Start the backend containers for Clickhouse:
+```
+docker compose up -d
+```
+
+Run RITA:
+    * Using Go:
+        ```
+        go run main.go <command> <flags>
+        ```
+    * Using a Compiled Binary and Versioning:
+        ```
+        make
+        ./rita <command> <flags>
+        ```
+
+
+#### Docker Development
+The installed version of RITA uses the `rita.sh` script to run RITA in Docker. For more fine-grained control of the build process, see the following:
+
+Build the RITA image:
+```
+sudo docker compose -f docker-compose.yml build
+```
+If this refuses to build, you may need to tell Docker what platform you're on:
+```
+export DOCKER_DEFAULT_PLATFORM=linux/arm64 && sudo docker compose -f docker-compose.prod.yml build
+```
+Reload the containers
+```
+docker compose -f docker-compose.prod.yml up -d
+```
+Run RITA:
+```
+docker compose -f docker-compose.prod.yml run --rm -it rita .....
+```

docs/System Requirements.md

@@ -0,0 +1,39 @@
+# System Requirements
+
+## Operating System
+
+- ✅: Official Support
+- ⚠️: Unofficial Support
+- ❌: Unsupported
+
+| OS              | Versions | Platform | Status | 
+| :---------------- | :------ | :---- | :----: | 
+| CentOS         | `9 Stream` | `amd64` | ✅ |
+| Rocky         | `9` | `amd64` | ✅ |
+| Ubuntu        |   `24.04`   | `amd64`| ✅ |
+| Windows        |    | | ❌ |
+
+## Hardware
+- CPU: RITA uses parallel processing and benefits from more CPU cores.
+- RAM: Larger datasets may require more memory.
+- Storage: RITA's datasets are significantly smaller than the Zeek logs so storage requirements are minimal compared to retaining the Zeek log files.
+
+### RITA
+The following are recommended specs for different use cases. 
+#### Casual Usage
+* Processor - 4+ cores. 
+* Memory - 16GB. 
+* Storage - SSD or NVME (250GB+)
+
+#### Production
+ * Processor - 8+ cores.
+ * Memory - 32GB
+ * Storage - SSD or NVME (500GB+)
+ 
+### Zeek (Production)
+The following requirements apply to the Zeek system.
+
+* Processor - Three cores plus an additional core for every 100 Mb of traffic being captured. This should be dedicated hardware, as resource congestion with other VMs can cause packets to be dropped or missed.
+* Memory - 16GB minimum. 64GB+ if monitoring 100Mb or more of network traffic. 128GB+ if monitoring 1Gb or more of network traffic.
+* Storage - 300GB minimum. 1TB+ is recommended to reduce log maintenance.
+* Network - In order to capture traffic with Zeek, you will need at least 2 network interface cards (NICs). One will be for management of the system and the other will be the dedicated capture port.

go.mod

@@ -0,0 +1,252 @@
+module activecm/rita
+
+go 1.22.3
+
+require (
+	github.com/ClickHouse/clickhouse-go/v2 v2.23.2
+	github.com/charmbracelet/bubbles v0.18.0
+	github.com/charmbracelet/bubbletea v0.26.3
+	github.com/charmbracelet/lipgloss v0.11.0
+	github.com/google/uuid v1.6.0
+	github.com/hjson/hjson-go/v4 v4.4.0
+	github.com/joho/godotenv v1.5.1
+	github.com/json-iterator/go v1.1.12
+	github.com/montanaflynn/stats v0.7.1
+	github.com/muesli/reflow v0.3.0
+	github.com/rs/zerolog v1.33.0
+	github.com/spf13/afero v1.11.0
+	github.com/stretchr/testify v1.9.0
+	github.com/testcontainers/testcontainers-go v0.31.0
+	github.com/testcontainers/testcontainers-go/modules/clickhouse v0.31.0
+	github.com/testcontainers/testcontainers-go/modules/compose v0.31.0
+	github.com/urfave/cli/v2 v2.27.2
+	github.com/vbauerster/mpb/v8 v8.7.3
+	golang.design/x/clipboard v0.7.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/text v0.15.0
+	golang.org/x/time v0.5.0
+)
+
+require github.com/google/go-querystring v1.1.0 // indirect
+
+require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
+	github.com/AlecAivazis/survey/v2 v2.3.7 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.11.5 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/atotto/clipboard v0.1.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver v3.5.1+incompatible
+	github.com/buger/goterm v1.0.4 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/charmbracelet/harmonica v0.2.0 // indirect
+	github.com/charmbracelet/x/ansi v0.1.2 // indirect
+	github.com/charmbracelet/x/input v0.1.1 // indirect
+	github.com/charmbracelet/x/term v0.1.1 // indirect
+	github.com/charmbracelet/x/windows v0.1.2 // indirect
+	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
+	github.com/compose-spec/compose-go/v2 v2.1.1 // indirect
+	github.com/containerd/console v1.0.4 // indirect
+	github.com/containerd/containerd v1.7.17 // indirect
+	github.com/containerd/continuity v0.4.3 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/ttrpc v1.2.4 // indirect
+	github.com/containerd/typeurl/v2 v2.1.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dchest/siphash v1.2.3
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/buildx v0.14.1 // indirect
+	github.com/docker/cli v26.1.3+incompatible // indirect
+	github.com/docker/compose/v2 v2.27.1 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v26.1.3+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.8.0 // indirect
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsevents v0.2.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/googleapis v1.4.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/in-toto/in-toto-golang v0.5.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jonboulle/clockwork v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/manifoldco/promptui v0.9.0
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mattn/go-shellwords v1.0.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/buildkit v0.13.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.7.1 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/signal v0.7.0 // indirect
+	github.com/moby/sys/symlink v0.2.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.15.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_golang v1.17.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sahilm/fuzzy v0.1.1 // indirect
+	github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect
+	github.com/shibumi/go-pathspec v1.3.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/theupdateframework/notary v0.7.0 // indirect
+	github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect
+	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
+	github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.uber.org/mock v0.4.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect
+	golang.org/x/exp/shiny v0.0.0-20240525044651-4c93da0ed11d // indirect
+	golang.org/x/image v0.16.0 // indirect
+	golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.15.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.20.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/grpc v1.59.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.29.2 // indirect
+	k8s.io/apimachinery v0.29.2 // indirect
+	k8s.io/apiserver v0.29.2 // indirect
+	k8s.io/client-go v0.29.2 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+	tags.cncf.io/container-device-interface v0.7.2 // indirect
+)

go.sum

@@ -0,0 +1,882 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y=
+cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
+cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU=
+github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
+github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
+github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2 h1:+DAKPMnxLS7pduQZsrJc8OhdLS2L9MfDEJ2TS+hpYDM=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2/go.mod h1:aNap51J1OM3yxQJRgM+AlP/MPkGBCL8A74uQThoQhR0=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38=
+github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU=
+github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
+github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
+github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc=
+github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
+github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
+github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
+github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o=
+github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
+github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
+github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0 h1:s7+5BfS4WFJoVF9pnB8kBk03S7pZXRdKamnV0FOl5Sc=
+github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0=
+github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw=
+github.com/charmbracelet/bubbletea v0.26.3 h1:iXyGvI+FfOWqkB2V07m1DF3xxQijxjY2j8PqiXYqasg=
+github.com/charmbracelet/bubbletea v0.26.3/go.mod h1:bpZHfDHTYJC5g+FBK+ptJRCQotRC+Dhh3AoMxa/2+3Q=
+github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
+github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
+github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g=
+github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8=
+github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY=
+github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
+github.com/charmbracelet/x/input v0.1.1 h1:YDOJaTUKCqtGnq9PHzx3pkkl4pXDOANUHmhH3DqMtM4=
+github.com/charmbracelet/x/input v0.1.1/go.mod h1:jvdTVUnNWj/RD6hjC4FsoB0SeZCJ2ZBkiuFP9zXvZI0=
+github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI=
+github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw=
+github.com/charmbracelet/x/windows v0.1.2 h1:Iumiwq2G+BRmgoayww/qfcvof7W/3uLoelhxojXlRWg=
+github.com/charmbracelet/x/windows v0.1.2/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ=
+github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ=
+github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
+github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/compose-spec/compose-go/v2 v2.1.1 h1:tKuYJwAVgxIryRrsvWJSf1kNviVOQVVqwyHsV6YoIUc=
+github.com/compose-spec/compose-go/v2 v2.1.1/go.mod h1:bEPizBkIojlQ20pi2vNluBa58tevvj0Y18oUSHPyfdc=
+github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
+github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
+github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=
+github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
+github.com/containerd/containerd v1.7.17 h1:KjNnn0+tAVQHAoaWRjmdak9WlvnFR/8rU1CHHy8Rm2A=
+github.com/containerd/containerd v1.7.17/go.mod h1:vK+hhT4TIv2uejlcDlbVIc8+h/BqtKLIyNrtCZol8lI=
+github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
+github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=
+github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/nydus-snapshotter v0.13.7 h1:x7DHvGnzJOu1ZPwPYkeOPk5MjZZYbdddygEjaSDoFTk=
+github.com/containerd/nydus-snapshotter v0.13.7/go.mod h1:VPVKQ3jmHFIcUIV2yiQ1kImZuBFS3GXDohKs9mRABVE=
+github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
+github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
+github.com/containerd/ttrpc v1.2.4 h1:eQCQK4h9dxDmpOb9QOOMh2NHTfzroH1IkmHiKZi05Oo=
+github.com/containerd/ttrpc v1.2.4/go.mod h1:ojvb8SJBSch0XkqNO0L0YX/5NxR3UnVk2LzFKBK0upc=
+github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4=
+github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E=
+github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
+github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
+github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/buildx v0.14.1 h1:Pr3HdtHoDsCghlIExgGp0WOIgvbiViushOKIPUIyFI4=
+github.com/docker/buildx v0.14.1/go.mod h1:s6xxLYXZIWnkdYpSvxRmoqZTb1vViV9q2f+Hg8cWA3Y=
+github.com/docker/cli v26.1.3+incompatible h1:bUpXT/N0kDE3VUHI2r5VMsYQgi38kYuoC0oL9yt3lqc=
+github.com/docker/cli v26.1.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/compose/v2 v2.27.1 h1:sOeqcMN7E8HmWpjD43OVZ+Fi2t96+ZGf3aK356mTgZE=
+github.com/docker/compose/v2 v2.27.1/go.mod h1:JZVWp9uVnP59S3KoVne6MboJRpx/eNr9HGE7/boB1vU=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo=
+github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8=
+github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40=
+github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
+github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
+github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
+github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
+github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
+github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
+github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsevents v0.2.0 h1:BRlvlqjvNTfogHfeBOFvSC9N0Ddy+wzQCQukyoD7o/c=
+github.com/fsnotify/fsevents v0.2.0/go.mod h1:B3eEk39i4hz8y1zaWS/wPrAP4O6wkIl7HQwKBr1qH/w=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
+github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-sql-driver/mysql v1.3.0 h1:pgwjLi/dvffoP9aabwkT3AKpXQM93QARkjFhDDqC1UE=
+github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
+github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
+github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
+github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 h1:jc2UWq7CbdszqeH6qu1ougXMIUBfSy8Pbh/anURYbGI=
+github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk=
+github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
+github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
+github.com/hjson/hjson-go/v4 v4.4.0 h1:D/NPvqOCH6/eisTb5/ztuIS8GUvmpHaLOcNk1Bjr298=
+github.com/hjson/hjson-go/v4 v4.4.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
+github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
+github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY=
+github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1GdHMCq8+WPxw8/BE=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE=
+github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc=
+github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
+github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
+github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.5.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
+github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
+github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
+github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
+github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/buildkit v0.13.2 h1:nXNszM4qD9E7QtG7bFWPnDI1teUQFQglBzon/IU3SzI=
+github.com/moby/buildkit v0.13.2/go.mod h1:2cyVOv9NoHM7arphK9ZfHIWKn9YVZRFd1wXB8kKmEzY=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
+github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
+github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
+github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
+github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
+github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
+github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
+github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI=
+github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg=
+github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc=
+github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs=
+github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
+github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
+github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
+github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
+github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
+github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
+github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU=
+github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o=
+github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
+github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
+github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE=
+github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU=
+github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
+github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
+github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
+github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
+github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
+github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
+github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
+github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
+github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
+github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
+github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY=
+github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94 h1:JmfC365KywYwHB946TTiQWEb8kqPY+pybPLoGE9GgVk=
+github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cobra v0.0.1/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431 h1:XTHrT015sxHyJ5FnQ0AeemSspZWaDq7DoTRW0EVsDCE=
+github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c h1:2EejZtjFjKJGk71ANb+wtFK5EjUzUkEM3R0xnp559xg=
+github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/testcontainers/testcontainers-go v0.31.0 h1:W0VwIhcEVhRflwL9as3dhY6jXjVCA27AkmbnZ+UTh3U=
+github.com/testcontainers/testcontainers-go v0.31.0/go.mod h1:D2lAoA0zUFiSY+eAflqK5mcUx/A5hrrORaEQrd0SefI=
+github.com/testcontainers/testcontainers-go/modules/clickhouse v0.31.0 h1:105EZAwt9bMBYEP8gsEp/DDP1St+2C5owXRMBrzN5M8=
+github.com/testcontainers/testcontainers-go/modules/clickhouse v0.31.0/go.mod h1:k3Ci/1vTkugSX67/oU0lOHfuTPpSLKw5KMg6d61uESI=
+github.com/testcontainers/testcontainers-go/modules/compose v0.31.0 h1:H74o3HisnApIDQx7sWibGzOl/Oo0By8DjyVeUf3qd6I=
+github.com/testcontainers/testcontainers-go/modules/compose v0.31.0/go.mod h1:z1JAsvL2/pNFy40yJX0VX9Yk+hzOCIO5DydxBJHBbCY=
+github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
+github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
+github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c h1:+6wg/4ORAbnSoGDzg2Q1i3CeMcT/jjhye/ZfnBHy7/M=
+github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM=
+github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=
+github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk=
+github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 h1:Y/M5lygoNPKwVNLMPXgVfsRT40CSFKXCxuU8LoHySjs=
+github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc=
+github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI=
+github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM=
+github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts=
+github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk=
+github.com/vbauerster/mpb/v8 v8.7.3 h1:n/mKPBav4FFWp5fH4U0lPpXfiOmCEgl5Yx/NM3tKJA0=
+github.com/vbauerster/mpb/v8 v8.7.3/go.mod h1:9nFlNpDGVoTmQ4QvNjSLtwLmAFjwmq0XaAF26toHGNM=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
+github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
+github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0 h1:NmnYCiR0qNufkldjVvyQfZTHSdzeHoZ41zggMsdMcLM=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0/go.mod h1:UVAO61+umUsHLtYb8KXXRoHtxUkdOPkYidzW3gipRLQ=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I=
+go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA=
+go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0=
+go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.design/x/clipboard v0.7.0 h1:4Je8M/ys9AJumVnl8m+rZnIvstSnYj1fvzqYrU3TXvo=
+golang.design/x/clipboard v0.7.0/go.mod h1:PQIvqYO9GP29yINEfsEn5zSQKAz3UgXmZKzDA6dnq2E=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
+golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp/shiny v0.0.0-20240525044651-4c93da0ed11d h1:NRBIrtUw7ZKcccaGmvwwMUwGTFGx6tVCtB+etaxQE5Q=
+golang.org/x/exp/shiny v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:3F+MieQB7dRYLTmnncoFbb1crS5lfQoTfDgQy6K4N0o=
+golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
+golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b h1:WX7nnnLfCEXg+FmdYZPai2XuP3VqCP1HZVMST0n9DF0=
+golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b/go.mod h1:EiXZlVfUTaAyySFVJb9rsODuiO+WXu8HrUuySb7nYFw=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ=
+google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY=
+google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo=
+google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc=
+google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
+gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
+gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
+gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM=
+gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A=
+k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0=
+k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8=
+k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
+k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ=
+k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ=
+k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg=
+k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA=
+k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
+k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
+k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
+k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+tags.cncf.io/container-device-interface v0.7.2 h1:MLqGnWfOr1wB7m08ieI4YJ3IoLKKozEnnNYBtacDPQU=
+tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto=

importer/conn.go

@@ -0,0 +1,237 @@
+package importer
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer/zeektypes"
+	zerolog "activecm/rita/logger"
+	"activecm/rita/progressbar"
+	"activecm/rita/util"
+	"errors"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/google/uuid"
+)
+
+var errParseSrcDst = "unable to parse valid ip address pair from conn log entry, skipping entry"
+
+type ConnEntry struct {
+	ImportTime           time.Time        `ch:"import_time"`
+	ZeekUID              util.FixedString `ch:"zeek_uid"`
+	ImportID             util.FixedString `ch:"import_id"`
+	Filtered             bool             `ch:"filtered"`
+	Hash                 util.FixedString `ch:"hash"`
+	Timestamp            time.Time        `ch:"ts"`
+	Src                  net.IP           `ch:"src"`
+	Dst                  net.IP           `ch:"dst"`
+	SrcNUID              uuid.UUID        `ch:"src_nuid"`
+	DstNUID              uuid.UUID        `ch:"dst_nuid"`
+	SrcPort              uint16           `ch:"src_port"`
+	DstPort              uint16           `ch:"dst_port"`
+	MissingHostHeader    bool             `ch:"missing_host_header"`    // used to mark HTTP entries that have a missing host header
+	MissingHostUseragent string           `ch:"missing_host_useragent"` // useragent for connections that have a missing host header
+	Proto                string           `ch:"proto"`
+	Service              string           `ch:"service"`
+	Duration             float64          `ch:"duration"`
+	SrcLocal             bool             `ch:"src_local"`
+	DstLocal             bool             `ch:"dst_local"`
+	ICMPType             int              `ch:"icmp_type"`
+	ICMPCode             int              `ch:"icmp_code"`
+	SrcBytes             int64            `ch:"src_bytes"`
+	DstBytes             int64            `ch:"dst_bytes"`
+	SrcIPBytes           int64            `ch:"src_ip_bytes"`
+	DstIPBytes           int64            `ch:"dst_ip_bytes"`
+	SrcPackets           int64            `ch:"src_packets"`
+	DstPackets           int64            `ch:"dst_packets"`
+	ConnState            string           `ch:"conn_state"`
+	MissedBytes          int64            `ch:"missed_bytes"`
+	ZeekHistory          string           `ch:"zeek_history"`
+}
+
+type UniqueConn struct {
+	Hash      util.FixedString `ch:"hash"`
+	Src       net.IP           `ch:"src"`
+	Dst       net.IP           `ch:"dst"`
+	SrcNUID   uuid.UUID        `ch:"src_nuid"`
+	DstNUID   uuid.UUID        `ch:"dst_nuid"`
+	ConnCount uint64
+	ConnType  string
+}
+
+type ZeekUIDRecord struct {
+	UID               util.FixedString
+	Timestamp         time.Time
+	UsedByFQDNBeacon  bool
+	UsedByDNS         bool
+	LinkedToHTTPEntry bool
+	NumUsedByHTTP     int
+	Duration          float64
+	SrcBytes          int64
+	DstBytes          int64
+	SrcIPBytes        int64
+	DstIPBytes        int64
+	SrcPackets        int64
+	DstPackets        int64
+	ConnState         string
+	Proto             string
+	Service           string
+}
+
+// parseConn listens on a channel of raw conn/openconn log records, formats them and sends them to be written to the database
+// func parseConn(conn <-chan zeektypes.Conn, output chan<- database.Data, uconnMap cmap.ConcurrentMap[string, *UniqueConn], zeekUIDMap cmap.ConcurrentMap[string, *ZeekUIDRecord], importID util.FixedString, numConns *uint64) {
+func parseConn(conn <-chan zeektypes.Conn, output chan<- database.Data, importID util.FixedString, importTime time.Time, numConns *uint64) {
+	logger := zerolog.GetLogger()
+
+	// loop over raw conn/openconn channel
+	for c := range conn {
+
+		// parse raw record as a conn/openconn entry
+		entry, err := formatConnRecord(&c, importID, importTime)
+		if err != nil {
+			logger.Warn().Err(err).
+				Str("log_path", c.LogPath).
+				Str("zeek_uid", c.UID).
+				Str("timestamp", (time.Unix(int64(c.TimeStamp), 0)).String()).
+				Str("src", c.Source).
+				Str("dst", c.Destination).
+				Send()
+
+			continue
+		}
+
+		// entry was subject to filtering
+		if entry == nil {
+			continue
+		}
+
+		output <- entry // send to log writer
+		if !entry.Filtered {
+			atomic.AddUint64(numConns, 1) // increment record counter
+		}
+	}
+
+}
+
+// formatConnRecord takes a raw conn record and formats it into the structure needed by the database
+func formatConnRecord(parseConn *zeektypes.Conn, importID util.FixedString, importTime time.Time) (*ConnEntry, error) { // filter filter
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	// get source destination pair for connection record
+	src := parseConn.Source
+	dst := parseConn.Destination
+
+	// parse addresses into binary format
+	srcIP := net.ParseIP(src)
+	dstIP := net.ParseIP(dst)
+
+	// verify that both addresses were parsed successfully
+	if (srcIP == nil) || (dstIP == nil) {
+		return nil, errors.New(errParseSrcDst)
+	}
+
+	// check if the connection is an icmp connection
+	icmpType, icmpCode := -1, -1
+
+	if parseConn.Proto == "icmp" {
+		icmpType = parseConn.SourcePort
+		icmpCode = parseConn.DestinationPort
+	}
+
+	srcNUID := util.ParseNetworkID(srcIP, parseConn.AgentUUID)
+	dstNUID := util.ParseNetworkID(dstIP, parseConn.AgentUUID)
+
+	hash, err := util.NewFixedStringHash(srcIP.To16().String() + srcNUID.String() + dstIP.To16().String() + dstNUID.String())
+	if err != nil {
+		return nil, err
+	}
+
+	zeekUID, err := util.NewFixedStringHash(parseConn.UID)
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := cfg.Filter.FilterConnPair(srcIP, dstIP)
+
+	entry := &ConnEntry{
+		ImportTime:  importTime,
+		ZeekUID:     zeekUID,
+		Filtered:    filtered,
+		Hash:        hash,
+		Timestamp:   time.Unix(int64(parseConn.TimeStamp), 0),
+		ImportID:    importID,
+		Src:         srcIP,
+		Dst:         dstIP,
+		SrcNUID:     srcNUID,
+		DstNUID:     dstNUID,
+		SrcPort:     uint16(parseConn.SourcePort),
+		DstPort:     uint16(parseConn.DestinationPort),
+		ZeekHistory: parseConn.History,
+		MissedBytes: parseConn.MissedBytes,
+		Proto:       parseConn.Proto,
+		Service:     parseConn.Service,
+		Duration:    parseConn.Duration,
+		SrcLocal:    cfg.Filter.CheckIfInternal(srcIP),
+		DstLocal:    cfg.Filter.CheckIfInternal(dstIP),
+		ICMPType:    icmpType,
+		ICMPCode:    icmpCode,
+		SrcBytes:    parseConn.OrigBytes,
+		DstBytes:    parseConn.RespBytes,
+		SrcIPBytes:  parseConn.OrigIPBytes,
+		DstIPBytes:  parseConn.RespIPBytes,
+		SrcPackets:  parseConn.OrigPackets,
+		DstPackets:  parseConn.RespPackets,
+		ConnState:   parseConn.ConnState,
+	}
+
+	// conn is treated differently than the rest of the logs since some other logs might need to correlate
+	// the zeek_uid data for entries that would otherwise be filtered out;
+	// For example: proxy connections require linking via zeek uid, but if the conn record is filtered out, then
+	// it will never get populated
+	// Filter out from never included list before adding it to the uconn map to allow blocking subnets
+	// that could end up overcommitting memory
+	ignore := cfg.Filter.FilterConnPairForHTTP(srcIP, dstIP)
+	if ignore {
+		return nil, nil
+	}
+
+	return entry, nil
+}
+
+// writeUnfilteredConns copies connections from conn_tmp to conn that were not marked as being filtered
+func (importer *Importer) writeUnfilteredConns(progress *tea.Program, open bool, spinnerID int) error {
+
+	tmpTable := "conn_tmp"
+	table := "conn"
+	if open {
+		tmpTable = "openconn_tmp"
+		table = "openconn"
+	}
+	chCtx := importer.Database.QueryParameters(clickhouse.Parameters{
+		"tmp_table": tmpTable,
+		"table":     table,
+	})
+
+	err := importer.Database.Conn.Exec(chCtx, `
+		INSERT INTO {table:Identifier} (
+			import_time, import_id, zeek_uid, hash, ts, src, dst, src_nuid, dst_nuid,
+			src_port, dst_port, missing_host_header, missing_host_useragent, proto, service,
+			conn_state, duration, src_local, dst_local, icmp_type, icmp_code, src_bytes, dst_bytes,
+			src_ip_bytes, dst_ip_bytes, src_packets, dst_packets, missed_bytes, zeek_history
+		) SELECT import_time, import_id, zeek_uid, hash, ts, src, dst, src_nuid, dst_nuid,
+			src_port, dst_port, missing_host_header, missing_host_useragent, proto, service,
+			conn_state, duration, src_local, dst_local, icmp_type, icmp_code, src_bytes, dst_bytes,
+			src_ip_bytes, dst_ip_bytes, src_packets, dst_packets, missed_bytes, zeek_history
+		FROM {tmp_table:Identifier}
+		WHERE filtered = false
+	`)
+
+	progress.Send(progressbar.ProgressSpinnerMsg(spinnerID))
+	return err
+}

importer/dns.go

@@ -0,0 +1,208 @@
+package importer
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer/zeektypes"
+	"activecm/rita/util"
+	"errors"
+	"net"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+var errMissingQuery = "blank or missing query field in dns log entry, skipping entry"
+
+type DNSEntry struct {
+	ImportTime          time.Time        `ch:"import_time"`
+	ZeekUID             util.FixedString `ch:"zeek_uid"`
+	Hash                util.FixedString `ch:"hash"`
+	Timestamp           time.Time        `ch:"ts"`
+	Src                 net.IP           `ch:"src"`
+	Dst                 net.IP           `ch:"dst"`
+	SrcNUID             uuid.UUID        `ch:"src_nuid"`
+	DstNUID             uuid.UUID        `ch:"dst_nuid"`
+	SrcPort             uint16           `ch:"src_port"`
+	DstPort             uint16           `ch:"dst_port"`
+	SrcLocal            bool             `ch:"src_local"`
+	DstLocal            bool             `ch:"dst_local"`
+	TransactionID       uint16           `ch:"transaction_id"`
+	RoundTripTime       float64          `ch:"round_trip_time"`
+	Query               string           `ch:"query"`
+	QueryClassCode      uint16           `ch:"query_class_code"`
+	QueryClassName      string           `ch:"query_class_name"`
+	QueryTypeCode       uint16           `ch:"query_type_code"`
+	QueryTypeName       string           `ch:"query_type_name"`
+	ResponseCode        uint16           `ch:"response_code"`
+	ResponseCodeName    string           `ch:"response_code_name"`
+	AuthoritativeAnswer bool             `ch:"authoritative_answer"`
+	RecursionDesired    bool             `ch:"recursion_desired"`
+	RecursionAvailable  bool             `ch:"recursion_available"`
+	Z                   uint16           `ch:"z"`
+	Answers             []string         `ch:"answers"`
+	TTLs                []float64        `ch:"ttls"`
+	Rejected            bool             `ch:"rejected"`
+	// PDNS field
+	ResolvedIP net.IP `ch:"resolved_ip"`
+}
+
+type UniqueFQDN struct {
+	Hash    util.FixedString `ch:"hash"`
+	Src     net.IP           `ch:"src"`
+	Dst     net.IP           `ch:"dst"`
+	FQDN    string           `ch:"fqdn"`
+	SrcNUID uuid.UUID        `ch:"src_nuid"`
+	DstNUID uuid.UUID        `ch:"dst_nuid"`
+}
+
+// parseDNS listens on a channel of raw dns log records, formats them into dns and pdns entries and and sends them to be written to the database
+func parseDNS(dns <-chan zeektypes.DNS, dnsOutput, pdnsOutput chan<- database.Data, numDNS, numPDNSRaw *uint64, importTime time.Time) {
+	// logger := zlog.GetLogger()
+
+	// loop over raw dns channel
+	for d := range dns {
+
+		// parse raw record as a dns entry
+		entry, err := formatDNSRecord(&d, importTime)
+		if err != nil {
+			// logger.Warn().Err(err).
+			// 	Str("log_path", d.LogPath).
+			// 	Str("zeek_uid", d.UID).
+			// 	Str("timestamp", (time.Unix(int64(d.TimeStamp), 0)).String()).
+			// 	Str("src", d.Source).
+			// 	Str("dst", d.Destination).
+			// 	Str("query", d.Query).
+			// 	Send()
+			continue
+		}
+
+		// entry was subject to filtering
+		if entry == nil {
+			continue
+		}
+
+		dnsOutput <- entry // send to dns log writer
+
+		// addToUDNS(uDNSMap, entry)   // add to unique dns map
+		atomic.AddUint64(numDNS, 1) // increment dns record counter
+
+		// parse dns entry into pdns entries based on dns entries's resolved ips
+		parsePDNSRecord(entry, pdnsOutput, numPDNSRaw)
+
+	}
+}
+
+// formatDNSRecord takes a raw dns record and formats it into the structure needed by the database
+func formatDNSRecord(parseDNS *zeektypes.DNS, importTime time.Time) (*DNSEntry, error) {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	// get source destination pair
+	src := parseDNS.Source
+	dst := parseDNS.Destination
+
+	// parse addresses into binary format
+	srcIP := net.ParseIP(src)
+	dstIP := net.ParseIP(dst)
+
+	// verify that both addresses were able to be parsed successfully
+	if (srcIP == nil) || (dstIP == nil) {
+		return nil, errors.New(errParseSrcDst)
+	}
+
+	// verify that query field is set
+	if parseDNS.Query == "" {
+		return nil, errors.New(errMissingQuery)
+	}
+
+	// ignore domains that have no periods (com, org, uk)
+	if !strings.Contains(parseDNS.Query, ".") {
+		return nil, nil
+	}
+
+	// Run query through filter to filter out certain domains and
+	// filter out traffic which is external -> external or external -> internal (if specified in the config file)
+	ignore := (cfg.Filter.FilterDomain(parseDNS.Query) || cfg.Filter.FilterDNSPair(srcIP, dstIP))
+
+	// If domain is not subject to filtering, process
+	if ignore {
+		return nil, nil
+	}
+
+	srcNUID := util.ParseNetworkID(srcIP, parseDNS.AgentUUID)
+	dstNUID := util.ParseNetworkID(dstIP, parseDNS.AgentUUID)
+
+	zeekUID, err := util.NewFixedStringHash(parseDNS.UID)
+	if err != nil {
+		return nil, err
+	}
+
+	hash, err := util.NewFixedStringHash(srcIP.To16().String(), dstIP.To16().String(), parseDNS.Query)
+	if err != nil {
+		return nil, err
+	}
+
+	entry := &DNSEntry{
+		ImportTime:          importTime,
+		ZeekUID:             zeekUID,
+		Hash:                hash,
+		Timestamp:           time.Unix(int64(parseDNS.TimeStamp), 0),
+		Src:                 srcIP,
+		Dst:                 dstIP,
+		SrcNUID:             srcNUID,
+		DstNUID:             dstNUID,
+		SrcPort:             uint16(parseDNS.SourcePort),
+		DstPort:             uint16(parseDNS.DestinationPort),
+		SrcLocal:            cfg.Filter.CheckIfInternal(srcIP),
+		DstLocal:            cfg.Filter.CheckIfInternal(dstIP),
+		TransactionID:       uint16(parseDNS.TransID),
+		RoundTripTime:       parseDNS.RTT,
+		Query:               parseDNS.Query,
+		QueryClassCode:      uint16(parseDNS.QClass),
+		QueryClassName:      parseDNS.QClassName,
+		QueryTypeCode:       uint16(parseDNS.QType),
+		QueryTypeName:       parseDNS.QTypeName,
+		ResponseCode:        uint16(parseDNS.RCode),
+		ResponseCodeName:    parseDNS.RCodeName,
+		AuthoritativeAnswer: parseDNS.AA,
+		RecursionDesired:    parseDNS.RD,
+		RecursionAvailable:  parseDNS.RA,
+		Z:                   uint16(parseDNS.Z),
+		Answers:             parseDNS.Answers,
+		TTLs:                parseDNS.TTLs,
+		Rejected:            parseDNS.Rejected,
+	}
+
+	return entry, nil
+}
+
+// parsePDNSRecord takes a single dns entry and splits it into multiple entries, one for each answer with a resolved ip in the dns record.
+func parsePDNSRecord(dnsRecord *DNSEntry, writeChan chan<- database.Data, numDNS *uint64) {
+
+	uniqueResolvedMap := make(map[string]bool)
+
+	if dnsRecord.QueryTypeName == "A" {
+		// attempt to parse answers, copy the entry for every resolved IP
+
+		// storing resolved IPs as an IPv6 column instead of an Array(IPv6) column significantly improves the
+		// lookup time of resolved IPs
+		for _, answer := range dnsRecord.Answers {
+			answerIP := net.ParseIP(answer)
+			// Check if answer is an IP address and store it if it is
+			if answerIP != nil {
+				uniqueResolvedMap[answer] = true
+				// we must create a copy of this entry by dereferencing it before
+				// assigning the resolved IP and sending it to the writer
+				newEntry := *dnsRecord
+				newEntry.ResolvedIP = answerIP.To16()
+				writeChan <- &newEntry
+				atomic.AddUint64(numDNS, 1) // increment pdns counter
+			}
+		}
+	}
+}

importer/http.go

@@ -0,0 +1,351 @@
+package importer
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer/zeektypes"
+	"activecm/rita/logger"
+	"activecm/rita/progressbar"
+	"activecm/rita/util"
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/google/uuid"
+)
+
+type HTTPEntry struct {
+	ImportTime   time.Time        `ch:"import_time"`
+	ZeekUID      util.FixedString `ch:"zeek_uid"`
+	Hash         util.FixedString `ch:"hash"`
+	Timestamp    time.Time        `ch:"ts"`
+	Src          net.IP           `ch:"src"`
+	Dst          net.IP           `ch:"dst"`
+	SrcNUID      uuid.UUID        `ch:"src_nuid"`
+	DstNUID      uuid.UUID        `ch:"dst_nuid"`
+	MultiRequest bool             `ch:"multi_request"`
+	SrcPort      uint16           `ch:"src_port"`
+	DstPort      uint16           `ch:"dst_port"`
+	Duration     float64          `ch:"duration"`
+	SrcLocal     bool             `ch:"src_local"`
+	DstLocal     bool             `ch:"dst_local"`
+	SrcBytes     int64            `ch:"src_bytes"`
+	DstBytes     int64            `ch:"dst_bytes"`
+	SrcIPBytes   int64            `ch:"src_ip_bytes"`
+	DstIPBytes   int64            `ch:"dst_ip_bytes"`
+	SrcPackets   int64            `ch:"src_packets"`
+	DstPackets   int64            `ch:"dst_packets"`
+	Proto        string           `ch:"proto"`
+	Service      string           `ch:"service"`
+	ConnState    string           `ch:"conn_state"`
+	TransDepth   uint16           `ch:"trans_depth"`
+	Method       string           `ch:"method"`
+	Host         string           `ch:"host"`
+	URI          string           `ch:"uri"`
+	Referrer     string           `ch:"referrer"`
+	HTTPVersion  string           `ch:"http_version"`
+	UserAgent    string           `ch:"useragent"`
+	Origin       string           `ch:"origin"`
+	StatusCode   int64            `ch:"status_code"`
+	StatusMsg    string           `ch:"status_msg"`
+	InfoCode     int64            `ch:"info_code"`
+	InfoMsg      string           `ch:"info_msg"`
+	Username     string           `ch:"username"`
+	Password     string           `ch:"password"`
+	SrcFUIDs     []string         `ch:"src_fuids"`
+	SrcFileNames []string         `ch:"src_file_names"`
+	SrcMIMETypes []string         `ch:"src_mime_types"`
+	DstFUIDs     []string         `ch:"dst_fuids"`
+	DstFileNames []string         `ch:"dst_file_names"`
+	DstMIMETypes []string         `ch:"dst_mime_types"`
+}
+
+// parseHTTP listens on a channel of raw http/openhttp log records, formats them and sends them to be linked with conn/openconn records and written to the database
+// func parseHTTP(http <-chan zeektypes.HTTP, zeekUIDMap cmap.ConcurrentMap[string, *ZeekUIDRecord], uHTTPMap cmap.ConcurrentMap[string, *UniqueFQDN], output chan database.Data, trackUIDLock *sync.Mutex, numHTTP *uint64) {
+func parseHTTP(http <-chan zeektypes.HTTP, output chan database.Data, connOutput chan database.Data, importID util.FixedString, trackUIDLock *sync.Mutex, importTime time.Time, numHTTP *uint64, numConn *uint64) {
+	logger := logger.GetLogger()
+
+	// loop over raw http/openhttp channel
+	for h := range http {
+
+		// parse raw record as an http/open http entry
+		entry, err := formatHTTPRecord(&h, importTime)
+		if err != nil {
+			logger.Warn().Err(err).
+				Str("log_path", h.LogPath).
+				Str("zeek_uid", h.UID).
+				Str("timestamp", (time.Unix(int64(h.TimeStamp), 0)).String()).
+				Str("src", h.Source).
+				Str("dst", h.Destination).
+				Str("fqdn", h.Host).
+				Str("uri", h.URI).
+				Send()
+			continue
+		}
+
+		// entry was subject to filtering
+		if entry == nil {
+			continue
+		}
+
+		if entry.Host == "" {
+			atomic.AddUint64(numConn, 1)
+		} else {
+			atomic.AddUint64(numHTTP, 1)
+		}
+
+		output <- entry
+	}
+
+}
+
+// formatHTTPRecord takes a raw http record and formats it into the structure needed by the database
+func formatHTTPRecord(parseHTTP *zeektypes.HTTP, importTime time.Time) (*HTTPEntry, error) {
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+	// get source destination pair for connection record
+	src := parseHTTP.Source
+	dst := parseHTTP.Destination
+
+	// parse addresses into binary format
+	srcIP := net.ParseIP(src)
+	dstIP := net.ParseIP(dst)
+
+	// verify that both addresses were able to be parsed successfully
+	if (srcIP == nil) || (dstIP == nil) {
+		return nil, errors.New(errParseSrcDst)
+	}
+
+	// parse host
+	fqdn := parseHTTP.Host
+
+	// check if destination is a proxy server based on HTTP method
+	dstIsProxy := (parseHTTP.Method == "CONNECT")
+
+	// if the HTTP method is CONNECT, then the srcIP is communicating
+	// to an FQDN through the dstIP proxy. We need to handle that
+	// as a special case here so that we don't filter internal->internal
+	// connections if the dstIP is an internal IP because the dstIP
+	// is an intermediary and not the final destination.
+	//
+	// The dstIP filter check is not included for proxy connections either
+	// because it isn't really the destination and it doesn't seem to make
+	// sense in this context to check for it. If the proxy IP is external,
+	// this will also allow a user to filter results from other modules
+	// (e.g., beacons), where false positives might arise due to the proxy IP
+	// appearing as a destination, while still allowing for processing that
+	// data for the proxy modules
+
+	srcLocal := cfg.Filter.CheckIfInternal(srcIP)
+	dstLocal := cfg.Filter.CheckIfInternal(dstIP)
+
+	if dstIsProxy {
+
+		if cfg.Filter.FilterDomain(fqdn) || cfg.Filter.FilterSingleIP(srcIP) {
+
+			return nil, nil
+		}
+		fqdnAsIPAddress := net.ParseIP(fqdn)
+
+		if fqdnAsIPAddress != nil && dstLocal && cfg.Filter.FilterConnPair(srcIP, fqdnAsIPAddress) {
+			return nil, nil
+		}
+	} else if cfg.Filter.FilterDomain(fqdn) || cfg.Filter.FilterConnPair(srcIP, dstIP) ||
+		// filter out connections where the src is external if the host isn't missing
+		(cfg.Filter.FilterSNIPair(srcIP) && parseHTTP.Host != "") {
+		return nil, nil
+	}
+
+	srcNUID := util.ParseNetworkID(srcIP, parseHTTP.AgentUUID)
+	dstNUID := util.ParseNetworkID(dstIP, parseHTTP.AgentUUID)
+
+	zeekUID, err := util.NewFixedStringHash(parseHTTP.UID)
+	if err != nil {
+		return nil, err
+	}
+
+	hash, err := util.NewFixedStringHash(srcIP.To16().String(), srcNUID.String(), dstIP.To16().String(), dstNUID.String(), fqdn)
+	if err != nil {
+		return nil, err
+	}
+
+	entry := &HTTPEntry{
+		ImportTime:   importTime,
+		ZeekUID:      zeekUID,
+		Hash:         hash,
+		Timestamp:    time.Unix(int64(parseHTTP.TimeStamp), 0),
+		Src:          srcIP,
+		Dst:          dstIP,
+		SrcNUID:      srcNUID,
+		DstNUID:      dstNUID,
+		SrcPort:      uint16(parseHTTP.SourcePort),
+		DstPort:      uint16(parseHTTP.DestinationPort),
+		SrcLocal:     srcLocal,
+		DstLocal:     dstLocal,
+		TransDepth:   uint16(parseHTTP.TransDepth),
+		Method:       parseHTTP.Method,
+		Host:         fqdn,
+		URI:          parseHTTP.URI,
+		Referrer:     parseHTTP.Referrer,
+		HTTPVersion:  parseHTTP.Version,
+		UserAgent:    parseHTTP.UserAgent,
+		Origin:       parseHTTP.Origin,
+		StatusCode:   parseHTTP.StatusCode,
+		StatusMsg:    parseHTTP.StatusMsg,
+		InfoCode:     parseHTTP.InfoCode,
+		InfoMsg:      parseHTTP.InfoMsg,
+		Username:     parseHTTP.UserName,
+		Password:     parseHTTP.Password,
+		SrcFUIDs:     parseHTTP.OrigFuids,
+		SrcFileNames: parseHTTP.OrigFilenames,
+		SrcMIMETypes: parseHTTP.OrigMimeTypes,
+		DstFUIDs:     parseHTTP.RespFuids,
+		DstFileNames: parseHTTP.RespFilenames,
+		DstMIMETypes: parseHTTP.RespMimeTypes,
+	}
+
+	return entry, nil
+}
+
+func (importer *Importer) writeLinkedHTTP(ctx context.Context, progress *tea.Program, barID int, httpWriter, connWriter *database.BulkWriter, open bool) error { //httpWriter chan database.Data, connWriter chan database.Data
+	logger := logger.GetLogger()
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	tmpTable := "http_tmp"
+	tableB := "conn_tmp"
+	if open {
+		tmpTable = "openhttp_tmp"
+		tableB = "openconn_tmp"
+	}
+
+	chCtx := importer.Database.QueryParameters(clickhouse.Parameters{
+		"tmp_table": tmpTable,
+		"table_b":   tableB,
+	})
+
+	var totalHTTP uint64
+	err = importer.Database.Conn.QueryRow(chCtx, `
+		SELECT count() FROM {tmp_table:Identifier}
+	`).Scan(&totalHTTP)
+	if err != nil {
+		return err
+	}
+
+	rows, err := importer.Database.Conn.Query(chCtx, `
+	WITH http_base AS (
+		SELECT zeek_uid, ts, src, src_nuid, dst, dst_nuid, src_port, dst_port, host, src_local, dst_local, useragent, method,
+			   uri, referrer, http_version, trans_depth, origin, 
+			   status_code, status_msg, info_code, info_msg, username, password,
+			   src_fuids, src_file_names, src_mime_types, dst_fuids, dst_mime_types,
+			   row_number() OVER (PARTITION BY zeek_uid ORDER BY ts DESC) AS rn
+		FROM {tmp_table:Identifier}
+	) 
+	SELECT 
+		h.zeek_uid as zeek_uid, c.hash as hash, c.ts AS ts, h.src_local as src_local, h.dst_local as dst_local,
+		h.src as src, h.src_nuid as src_nuid, h.dst as dst, h.dst_nuid as dst_nuid, h.src_port as src_port, h.dst_port as dst_port,
+		h.host as host, h.useragent as useragent, h.method as method, h.uri as uri, h.referrer as referrer, h.http_version as http_version,
+		h.trans_depth as trans_depth, h.origin as origin, h.status_code as status_code, h.status_msg as status_msg, h.info_code as info_code,
+		h.info_msg as info_msg, h.username as username, h.password as password, h.src_fuids as src_fuids,
+		h.src_file_names as src_file_names, h.src_mime_types as src_mime_types, h.dst_fuids as dst_fuids, h.dst_mime_types as dst_mime_types,
+		-- set proto and service regardless of whether it was linked already or not
+		-- since multi-requests can use different dst ports and still have the same UID, so
+		-- it is useful to be able to see the dst ports coming from multi request entries as well
+		c.proto as proto, c.service as service,
+		if( h.rn = 1, c.src_ip_bytes, 0) as src_ip_bytes,
+		if( h.rn = 1, c.dst_ip_bytes, 0) as dst_ip_bytes,
+		if( h.rn = 1, c.src_bytes, 0) as src_bytes,
+		if( h.rn = 1, c.dst_bytes, 0) as dst_bytes,
+		if( h.rn = 1, c.duration, 0) as duration,
+		if( h.rn = 1, c.conn_state, '') as conn_state,
+		if( h.rn = 1, c.src_packets, 0) as src_packets,
+		if( h.rn = 1, c.dst_packets, 0) as dst_packets,
+		if( h.rn > 1,true,  0) as multi_request
+	FROM http_base h
+	INNER JOIN {table_b:Identifier} c USING zeek_uid
+	WHERE h.rn <= 20
+`)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for rows.Next() {
+		select {
+		// abort this function if the context was cancelled
+		case <-ctx.Done():
+			logger.Warn().Msg("cancelling HTTP connection linking")
+			rows.Close()
+			return ctx.Err()
+		default:
+			var entry HTTPEntry
+
+			err := rows.ScanStruct(&entry)
+			if err != nil {
+				return err
+			}
+			i++
+			// update progress bar every 1000 entries
+			if i%1000 == 0 {
+				progress.Send(progressbar.ProgressMsg{ID: barID, Percent: float64(float64(i) / float64(totalHTTP))})
+			}
+			entry.ImportTime = importer.Database.ImportStartedAt
+
+			hash, err := util.NewFixedStringHash(entry.Src.To16().String(), entry.SrcNUID.String(), entry.Host)
+			if err != nil {
+				return err
+			}
+
+			switch {
+			case entry.Host == "":
+				ignore := cfg.Filter.FilterConnPair(entry.Src, entry.Dst)
+				if ignore {
+					continue
+				}
+
+				icmpType, icmpCode := -1, -1
+
+				if entry.Proto == "icmp" {
+					icmpType = int(entry.SrcPort)
+					icmpCode = int(entry.DstPort)
+				}
+
+				connEntry := &ConnEntry{
+					ZeekUID:              entry.ZeekUID,
+					ImportID:             importer.ImportID,
+					ImportTime:           entry.ImportTime,
+					Hash:                 entry.Hash,
+					Timestamp:            entry.Timestamp,
+					Src:                  entry.Src,
+					Dst:                  entry.Dst,
+					SrcNUID:              entry.SrcNUID,
+					DstNUID:              entry.DstNUID,
+					SrcPort:              entry.SrcPort,
+					DstPort:              entry.DstPort,
+					MissingHostHeader:    true,            // this field MUST be set to true
+					MissingHostUseragent: entry.UserAgent, // this field MUST be set
+					SrcLocal:             entry.SrcLocal,
+					DstLocal:             entry.DstLocal,
+					ICMPType:             icmpType,
+					ICMPCode:             icmpCode,
+				}
+				connWriter.WriteChannel <- connEntry
+			default:
+				entry.Hash = hash
+				httpWriter.WriteChannel <- &entry
+			}
+		}
+	}
+	rows.Close()
+	progress.Send(progressbar.ProgressMsg{ID: barID, Percent: 1})
+
+	return nil
+}

importer/importer.go

@@ -0,0 +1,676 @@
+package importer
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer/zeektypes"
+	zerolog "activecm/rita/logger"
+	"activecm/rita/progressbar"
+	"activecm/rita/util"
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/charmbracelet/bubbles/progress"
+	"github.com/spf13/afero"
+	"github.com/vbauerster/mpb/v8"
+	"github.com/vbauerster/mpb/v8/decor"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+	"golang.org/x/time/rate"
+)
+
+var errAllFilesPreviouslyImported = errors.New("all files were previously imported")
+
+type zeekRecord interface {
+	zeektypes.Conn | zeektypes.DNS | zeektypes.HTTP | zeektypes.SSL
+}
+
+type Importer struct {
+	Database                 *database.DB
+	ImportID                 util.FixedString
+	LogDirectory             string
+	FileMap                  map[string][]string
+	EntryChannels            EntryChans
+	MetaDBChannel            chan MetaDBFile
+	Paths                    chan string
+	ErrChannel               chan error
+	TotalFileCount           int
+	DoneChannels             DoneChans
+	Writers                  writers
+	WriteLimiter             *rate.Limiter
+	ProgressBar              *mpb.Progress
+	FileProgressBar          *mpb.Bar
+	ProgressLogger           *log.Logger
+	HTTPLinkMutex            sync.Mutex
+	OpenHTTPLinkMutex        sync.Mutex
+	NumParsers               int
+	NumDigesters             int
+	NumWriters               int
+	ResultCounts             ResultCounts
+	wg                       WaitGroups
+	importStartedCallback    func(util.FixedString) error
+	validateLogFilesCallback func(map[string][]string) (int, error)
+	startWritersCallback     func(int)
+	closeWritersCallback     func()
+	markFileImportedCallback func(util.FixedString, util.FixedString, string) error
+}
+
+type EntryChans struct {
+	Conn     chan zeektypes.Conn
+	OpenConn chan zeektypes.Conn
+	DNS      chan zeektypes.DNS
+	HTTP     chan zeektypes.HTTP
+	OpenHTTP chan zeektypes.HTTP
+	SSL      chan zeektypes.SSL
+	OpenSSL  chan zeektypes.SSL
+}
+
+type writers struct {
+	ConnTmp     *database.BulkWriter
+	OpenConnTmp *database.BulkWriter
+	DNS         *database.BulkWriter
+	PDNS        *database.BulkWriter
+	HTTPTmp     *database.BulkWriter
+	OpenHTTPTmp *database.BulkWriter
+	SSLTmp      *database.BulkWriter
+	OpenSSLTmp  *database.BulkWriter
+}
+
+type DoneChans struct {
+	filesDone chan struct{}
+	conn      chan struct{}
+	openconn  chan struct{}
+	http      chan struct{}
+	openhttp  chan struct{}
+	dns       chan struct{}
+	ssl       chan struct{}
+	openssl   chan struct{}
+}
+
+type ResultCounts struct {
+	ZeekUIDs       uint64
+	OpenZeekUIDs   uint64
+	UnfilteredConn uint64
+	Conn           uint64
+	OpenConn       uint64
+	HTTP           uint64
+	OpenHTTP       uint64
+	DNS            uint64
+	UDNS           int64
+	PDNSRaw        uint64
+	SSL            uint64
+	OpenSSL        uint64
+}
+
+type WaitGroups struct {
+	Digester sync.WaitGroup
+	MetaDB   sync.WaitGroup
+	OpenConn sync.WaitGroup
+	Conn     sync.WaitGroup
+	DNS      sync.WaitGroup
+	HTTP     sync.WaitGroup
+	OpenHTTP sync.WaitGroup
+	SSL      sync.WaitGroup
+	OpenSSL  sync.WaitGroup
+}
+
+// NewImporter creates and returns a new Importer object
+func NewImporter(db *database.DB, cfg *config.Config, importStartedAt time.Time, numDigesters int, numParsers int, numWriters int) (*Importer, error) {
+	logger := zerolog.GetLogger()
+
+	// create channels to hold the network traffic entries
+	entryChannels := EntryChans{
+		Conn:     make(chan zeektypes.Conn, 1000),
+		OpenConn: make(chan zeektypes.Conn, 1000),
+		DNS:      make(chan zeektypes.DNS, 1000),
+		HTTP:     make(chan zeektypes.HTTP, 1000),
+		OpenHTTP: make(chan zeektypes.HTTP, 1000),
+		SSL:      make(chan zeektypes.SSL, 1000),
+		OpenSSL:  make(chan zeektypes.SSL, 1000),
+	}
+
+	// create channels to keep track of log files being successfully imported
+	doneChannels := DoneChans{
+		filesDone: make(chan struct{}),
+		conn:      make(chan struct{}, numDigesters),
+		openconn:  make(chan struct{}, numDigesters),
+		http:      make(chan struct{}, numDigesters),
+		openhttp:  make(chan struct{}, numDigesters),
+		dns:       make(chan struct{}, numDigesters),
+		ssl:       make(chan struct{}, numDigesters),
+		openssl:   make(chan struct{}, numDigesters),
+	}
+
+	// create a rate limiter to control the rate of writing to the database
+	limiter := rate.NewLimiter(5, 5)
+
+	// create writer objects to write output data to the individual log collections
+	writers := writers{
+		ConnTmp:     database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "conn_tmp", "INSERT INTO {database:Identifier}.conn_tmp", limiter, false),
+		OpenConnTmp: database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "openconn_tmp", "INSERT INTO {database:Identifier}.openconn_tmp", limiter, false),
+		DNS:         database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "dns", "INSERT INTO {database:Identifier}.dns", limiter, false),
+		PDNS:        database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "pdns", "INSERT INTO {database:Identifier}.pdns_raw", limiter, false),
+		HTTPTmp:     database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "http_tmp", "INSERT INTO {database:Identifier}.http_tmp", limiter, false),
+		OpenHTTPTmp: database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "openhttp_tmp", "INSERT INTO {database:Identifier}.openhttp_tmp", limiter, false),
+		SSLTmp:      database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "ssl_tmp", "INSERT INTO {database:Identifier}.ssl_tmp", limiter, false),
+		OpenSSLTmp:  database.NewBulkWriter(db, cfg, numWriters, db.GetSelectedDB(), "openssl_tmp", "INSERT INTO {database:Identifier}.openssl_tmp", limiter, false),
+	}
+
+	// create progress bar
+	progress := mpb.New(mpb.WithWidth(64))
+
+	// set the overall db import start time
+	db.ImportStartedAt = importStartedAt
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(importStartedAt.UnixMicro(), 10))
+	if err != nil {
+		return nil, err
+	}
+
+	// log the import id
+	logger.Debug().Str("import_id", importID.Hex()).Send()
+
+	// return the importer object
+	return &Importer{
+		Database: db,
+		ImportID: importID,
+		// LogDirectory:             directory,
+		FileMap:                  make(map[string][]string),
+		EntryChannels:            entryChannels,
+		MetaDBChannel:            make(chan MetaDBFile),
+		Paths:                    make(chan string, 10),
+		ErrChannel:               make(chan error, 100),
+		DoneChannels:             doneChannels,
+		Writers:                  writers,
+		WriteLimiter:             rate.NewLimiter(5, 5),
+		ProgressBar:              progress,
+		ProgressLogger:           log.New(progress, "", 0),
+		NumParsers:               numParsers,
+		NumDigesters:             numDigesters,
+		NumWriters:               numWriters,
+		ResultCounts:             ResultCounts{},
+		importStartedCallback:    db.AddImportStartRecordToMetaDB,
+		validateLogFilesCallback: db.CheckIfFilesWereAlreadyImported,
+		startWritersCallback:     writers.startWriters,
+		closeWritersCallback:     writers.closeWriters,
+		markFileImportedCallback: db.MarkFileImportedInMetaDB,
+	}, nil
+}
+
+func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error {
+	logger := zerolog.GetLogger()
+
+	// record the hourlyImportStart time of this import chunk
+	hourlyImportStart := time.Now()
+
+	// check if files have already been imported make a map of the remaining files
+	totalFileCount, err := importer.validateLogFilesCallback(files)
+	if err != nil {
+		return err
+	}
+
+	// verify that there are still files left to import and set file count
+	if totalFileCount < 1 {
+		return errAllFilesPreviouslyImported
+	}
+	importer.TotalFileCount = totalFileCount
+
+	// set up the file map with the remaining files
+	importer.FileMap = files
+
+	// add import started record to metadatabase
+	err = importer.importStartedCallback(importer.ImportID)
+	if err != nil {
+		return err
+	}
+
+	// initialize progress bar
+	importer.FileProgressBar = importer.ProgressBar.New(int64(importer.TotalFileCount),
+		mpb.BarStyle().Lbound("╢").Filler("▌").Tip("▌").Padding("░").Rbound("╟"),
+		mpb.PrependDecorators(
+			// display our name with one space on the right
+			decor.Name("Log Parsing", decor.WC{C: decor.DindentRight | decor.DextraSpace}),
+			// replace ETA decorator with "done" message, OnComplete event
+			decor.OnComplete(decor.Elapsed(decor.ET_STYLE_GO), "🎉"),
+		),
+		mpb.AppendDecorators(decor.CountersNoUnit("%d / %d")),
+	)
+
+	// start the import
+	importer.process(afs)
+
+	// record import time to logger
+	hourlyImportEnd := time.Now()
+	logger.Info().Time("parsing_began", hourlyImportStart).Time("parsing_finished", hourlyImportEnd).Str("elapsed_time", time.Since(hourlyImportStart).String()).Msg("Finished Parsing Logs! 🎉")
+
+	if err := importer.season(); err != nil {
+		return err
+	}
+	seasoningEnd := time.Now()
+	logger.Info().Time("seasoning_began", hourlyImportEnd).Time("seasoning_finished", seasoningEnd).Str("elapsed_time", time.Since(hourlyImportEnd).String()).Msg("Finished Seasoning Logs! 🎉")
+
+	// create formatter for adding commas in the counts
+	p := message.NewPrinter(language.English)
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.Conn)).Msg("Imported conn records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.OpenConn)).Msg("Imported open conn records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.DNS)).Msg("Imported dns records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.PDNSRaw)).Msg("Imported pdns raw records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.HTTP)).Msg("Imported http records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.OpenHTTP)).Msg("Imported open http records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.SSL)).Msg("Imported ssl records")
+	logger.Debug().Str("count", p.Sprintf("%d", importer.ResultCounts.OpenSSL)).Msg("Imported open ssl records")
+
+	return nil
+}
+
+// process loads the files and parses the raw log entries
+func (importer *Importer) process(afs afero.Fs) {
+	// initialize writers
+	importer.startWritersCallback(importer.NumWriters)
+
+	// start goroutines to write network traffic entries to the database
+	importer.startParseRoutines()
+
+	// start goroutines to parse log file contents
+	importer.startDigesters(afs)
+
+	// start goroutine to mark files as imported in MetaDB
+	importer.startMetaDBFileTracker()
+
+	// start listener routine and feed initial files in to paths channel
+	importer.feedAndListenForFileCompletion()
+
+	// close log entry and map channels
+	go func() {
+		// wait for our bar to complete and flush
+		importer.ProgressBar.Wait()
+
+		// close log entry channels
+		close(importer.EntryChannels.Conn)
+		close(importer.EntryChannels.OpenConn)
+		close(importer.EntryChannels.DNS)
+		close(importer.EntryChannels.HTTP)
+		close(importer.EntryChannels.OpenHTTP)
+		close(importer.EntryChannels.SSL)
+		close(importer.EntryChannels.OpenSSL)
+
+		// close paths channel
+		close(importer.Paths)
+		// close metadb channel
+		close(importer.MetaDBChannel)
+	}()
+
+	// wait for log routine groups
+	importer.wg.Conn.Wait()
+	importer.wg.OpenConn.Wait()
+	importer.wg.DNS.Wait()
+	importer.wg.HTTP.Wait()
+	importer.wg.OpenHTTP.Wait()
+	importer.wg.SSL.Wait()
+	importer.wg.OpenSSL.Wait()
+
+	close(importer.DoneChannels.conn)
+	close(importer.DoneChannels.openconn)
+	close(importer.DoneChannels.http)
+	close(importer.DoneChannels.openhttp)
+	close(importer.DoneChannels.ssl)
+	close(importer.DoneChannels.openssl)
+	close(importer.DoneChannels.dns)
+	close(importer.DoneChannels.filesDone)
+
+	close(importer.ErrChannel)
+
+	// close writers
+	importer.closeWritersCallback()
+}
+
+// startParseRoutines starts a fixed number of goroutines to parse lines from logs into data to be written to the db.
+func (importer *Importer) startParseRoutines() {
+	importer.wg.Conn.Add(importer.NumParsers)
+	importer.wg.OpenConn.Add(importer.NumParsers)
+	importer.wg.DNS.Add(importer.NumParsers)
+	importer.wg.HTTP.Add(importer.NumParsers)
+	importer.wg.OpenHTTP.Add(importer.NumParsers)
+	importer.wg.SSL.Add(importer.NumParsers)
+	importer.wg.OpenSSL.Add(importer.NumParsers)
+
+	for i := 0; i < importer.NumParsers; i++ {
+		go func(_ int) {
+			// parseConn(importer.EntryChannels.Conn, importer.Writers.Conn.WriteChannel, importer.UniqueMaps.Uconn, importer.UniqueMaps.ZeekUIDs, importer.ImportID, &importer.ResultCounts.Conn)
+			parseConn(importer.EntryChannels.Conn, importer.Writers.ConnTmp.WriteChannel, importer.ImportID, importer.Database.ImportStartedAt, &importer.ResultCounts.Conn)
+			importer.wg.Conn.Done()
+		}(i)
+		go func(_ int) {
+			// parseConn(importer.EntryChannels.OpenConn, importer.Writers.OpenConn.WriteChannel, importer.UniqueMaps.OpenConn, importer.UniqueMaps.OpenZeekUIDs, importer.ImportID, &importer.ResultCounts.OpenConn)
+			parseConn(importer.EntryChannels.OpenConn, importer.Writers.OpenConnTmp.WriteChannel, importer.ImportID, importer.Database.ImportStartedAt, &importer.ResultCounts.OpenConn)
+			importer.wg.OpenConn.Done()
+		}(i)
+
+		go func(_ int) {
+			parseDNS(importer.EntryChannels.DNS, importer.Writers.DNS.WriteChannel, importer.Writers.PDNS.WriteChannel, &importer.ResultCounts.DNS, &importer.ResultCounts.PDNSRaw, importer.Database.ImportStartedAt)
+			importer.wg.DNS.Done()
+		}(i)
+
+		go func(_ int) {
+			// parseHTTP(importer.EntryChannels.HTTP, importer.UniqueMaps.ZeekUIDs, importer.UniqueMaps.UHTTP, importer.Writers.HTTP.WriteChannel, &importer.HTTPLinkMutex, &importer.ResultCounts.HTTP)
+			parseHTTP(importer.EntryChannels.HTTP, importer.Writers.HTTPTmp.WriteChannel, importer.Writers.ConnTmp.WriteChannel, importer.ImportID, &importer.HTTPLinkMutex, importer.Database.ImportStartedAt, &importer.ResultCounts.HTTP, &importer.ResultCounts.Conn)
+			importer.wg.HTTP.Done()
+		}(i)
+
+		go func(_ int) {
+			// parseHTTP(importer.EntryChannels.OpenHTTP, importer.UniqueMaps.OpenZeekUIDs, importer.UniqueMaps.UOpenHTTP, importer.Writers.OpenHTTP.WriteChannel, &importer.OpenHTTPLinkMutex, &importer.ResultCounts.OpenHTTP)
+			parseHTTP(importer.EntryChannels.OpenHTTP, importer.Writers.OpenHTTPTmp.WriteChannel, importer.Writers.OpenConnTmp.WriteChannel, importer.ImportID, &importer.OpenHTTPLinkMutex, importer.Database.ImportStartedAt, &importer.ResultCounts.OpenHTTP, &importer.ResultCounts.OpenConn)
+			importer.wg.OpenHTTP.Done()
+		}(i)
+
+		go func(_ int) {
+			// parseSSL(importer.EntryChannels.SSL, importer.UniqueMaps.ZeekUIDs, importer.UniqueMaps.USSL, importer.Writers.SSL.WriteChannel, &importer.ResultCounts.SSL)
+			parseSSL(importer.EntryChannels.SSL, importer.Writers.SSLTmp.WriteChannel, importer.Database.ImportStartedAt, &importer.ResultCounts.SSL)
+			importer.wg.SSL.Done()
+		}(i)
+
+		go func(_ int) {
+			// parseSSL(importer.EntryChannels.OpenSSL, importer.UniqueMaps.OpenZeekUIDs, importer.UniqueMaps.UOpenSSL, importer.Writers.OpenSSL.WriteChannel, &importer.ResultCounts.OpenSSL)
+			parseSSL(importer.EntryChannels.OpenSSL, importer.Writers.OpenSSLTmp.WriteChannel, importer.Database.ImportStartedAt, &importer.ResultCounts.OpenSSL)
+			importer.wg.OpenSSL.Done()
+		}(i)
+	}
+}
+
+// startDigesters starts a fixed number of goroutines to read and digest files.
+func (importer *Importer) startDigesters(afs afero.Fs) {
+	importer.wg.Digester.Add(importer.NumDigesters)
+	for i := 0; i < importer.NumDigesters; i++ {
+		go func(_ int) {
+			digester(afs, importer.DoneChannels, importer.Paths, importer.ErrChannel, importer.EntryChannels, importer.MetaDBChannel, importer.Database.GetSelectedDB(), importer.ImportID, importer.ProgressLogger)
+			importer.wg.Digester.Done()
+		}(i)
+	}
+}
+
+// startMetaDBFileTracker starts a goroutine to mark files as imported in MetaDB
+func (importer *Importer) startMetaDBFileTracker() {
+
+	importer.wg.MetaDB.Add(1)
+	go func() {
+		for metaDB := range importer.MetaDBChannel {
+			err := importer.markFileImportedCallback(metaDB.fileHash, metaDB.importID, metaDB.path)
+			if err != nil {
+				importer.ProgressLogger.Println("[WARNING] could not mark file as imported, path:", metaDB.path, err)
+			}
+		}
+		importer.wg.MetaDB.Done()
+	}()
+
+}
+
+// feedAndListenForFileCompletion feeds files to the paths channel and listens for the completion of each log type
+// to orchestrate feeding other log types into the paths channel
+func (importer *Importer) feedAndListenForFileCompletion() {
+	// listen for file completion of each log type
+	go func() {
+		for importer.FileProgressBar.Current() < int64(importer.TotalFileCount) {
+
+			select {
+			// read from other log types' done channels so that they don't block
+			case <-importer.DoneChannels.conn:
+			case <-importer.DoneChannels.openconn:
+			case <-importer.DoneChannels.http:
+			case <-importer.DoneChannels.openhttp:
+			case <-importer.DoneChannels.ssl:
+			case <-importer.DoneChannels.openssl:
+			case <-importer.DoneChannels.dns:
+
+			// increment progress bar
+			case <-importer.DoneChannels.filesDone:
+				importer.FileProgressBar.Increment()
+			default:
+			}
+
+		}
+	}()
+
+	if len(importer.FileMap[ConnPrefix]) > 0 {
+		for _, connLog := range importer.FileMap[ConnPrefix] {
+			importer.Paths <- connLog
+		}
+		for _, httpLog := range importer.FileMap[HTTPPrefix] {
+			importer.Paths <- httpLog
+		}
+		for _, sslLog := range importer.FileMap[SSLPrefix] {
+			importer.Paths <- sslLog
+		}
+	}
+	if len(importer.FileMap[OpenConnPrefix]) > 0 {
+		for _, openConnLog := range importer.FileMap[OpenConnPrefix] {
+			importer.Paths <- openConnLog
+		}
+		for _, openHTTPLog := range importer.FileMap[OpenHTTPPrefix] {
+			importer.Paths <- openHTTPLog
+		}
+		for _, openSSLLog := range importer.FileMap[OpenSSLPrefix] {
+			importer.Paths <- openSSLLog
+		}
+	}
+	for _, dnsLog := range importer.FileMap[DNSPrefix] {
+		importer.Paths <- dnsLog
+	}
+}
+
+// digester loops over the paths, checks the file prefix, and sends each path to the parser with its corresponding entryChannel until either paths or done is closed.
+func digester(afs afero.Fs, done DoneChans, paths <-chan string, errc chan error, entryChannels EntryChans, metaDBChan chan<- MetaDBFile, database string, importID util.FixedString, progressLogger *log.Logger) {
+	// errc := make(chan error)
+
+	// read entries from err channel, handle specific errors if necessary
+	// currently, this err channel is primarily used for checking errors in tests
+	go func() {
+		for err := range errc {
+			_ = err
+		}
+	}()
+
+	// loop over paths and send to parseFiles with the correct corresponding entryChannels, sending a done signal for each completed file
+	for path := range paths {
+		progressLogger.Println("[-] Parsing: ", path)
+		switch {
+		case strings.HasPrefix(filepath.Base(path), ConnPrefix):
+			parseFile(afs, path, entryChannels.Conn, errc, metaDBChan, database, importID)
+			done.conn <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), OpenConnPrefix):
+			parseFile(afs, path, entryChannels.OpenConn, errc, metaDBChan, database, importID)
+			done.openconn <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), DNSPrefix):
+			parseFile(afs, path, entryChannels.DNS, errc, metaDBChan, database, importID)
+			done.dns <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), HTTPPrefix):
+			parseFile(afs, path, entryChannels.HTTP, errc, metaDBChan, database, importID)
+			done.http <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), OpenHTTPPrefix):
+			parseFile(afs, path, entryChannels.OpenHTTP, errc, metaDBChan, database, importID)
+			done.openhttp <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), SSLPrefix):
+			parseFile(afs, path, entryChannels.SSL, errc, metaDBChan, database, importID)
+			done.ssl <- struct{}{}
+		case strings.HasPrefix(filepath.Base(path), OpenSSLPrefix):
+			parseFile(afs, path, entryChannels.OpenSSL, errc, metaDBChan, database, importID)
+			done.openssl <- struct{}{}
+		}
+		done.filesDone <- struct{}{}
+	}
+}
+
+// startWriters start a fixed number of writer workers for each table to write to
+func (writer *writers) startWriters(numWriters int) {
+	for i := 0; i < numWriters; i++ {
+		writer.ConnTmp.Start(i)
+		writer.OpenConnTmp.Start(i)
+		writer.DNS.Start(i)
+		writer.PDNS.Start(i)
+		writer.HTTPTmp.Start(i)
+		writer.OpenHTTPTmp.Start(i)
+		writer.SSLTmp.Start(i)
+		writer.OpenSSLTmp.Start(i)
+	}
+}
+
+// closeWriters close each writer
+func (writer *writers) closeWriters() {
+	writer.ConnTmp.Close()
+	writer.OpenConnTmp.Close()
+	writer.DNS.Close()
+	writer.PDNS.Close()
+	writer.HTTPTmp.Close()
+	writer.OpenHTTPTmp.Close()
+	writer.SSLTmp.Close()
+	writer.OpenSSLTmp.Close()
+}
+
+// season links the http & ssl logs with the conn logs and adds data to those connections
+func (importer *Importer) season() error {
+	logger := zerolog.GetLogger()
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return err
+	}
+
+	limiter := rate.NewLimiter(5, 5)
+	writerWorkers := 2
+	sslWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "ssl", "INSERT INTO {database:Identifier}.ssl", limiter, false)
+	openSSLWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "openssl", "INSERT INTO {database:Identifier}.openssl", limiter, false)
+	httpWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "http", "INSERT INTO {database:Identifier}.http", limiter, false)
+	connWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "conn", "INSERT INTO {database:Identifier}.conn", limiter, false)
+	openHTTPWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "openhttp", "INSERT INTO {database:Identifier}.openhttp", limiter, false)
+	openConnWriter := database.NewBulkWriter(importer.Database, cfg, writerWorkers, importer.Database.GetSelectedDB(), "openconn", "INSERT INTO {database:Identifier}.openconn", limiter, false)
+
+	for i := 0; i < writerWorkers; i++ {
+		sslWriter.Start(i)
+		openSSLWriter.Start(i)
+		httpWriter.Start(i)
+		openHTTPWriter.Start(i)
+		connWriter.Start(i)
+		openConnWriter.Start(i)
+	}
+
+	linkingErrGroup, ctx := errgroup.WithContext(context.Background())
+
+	gradient := progress.WithGradient("#f67a70", "#ffda6d")
+
+	var barList []*progressbar.ProgressBar
+	var spinners []progressbar.Spinner
+	sslBarName := "🧂 Seasoning SSL connections "
+	httpBarName := "🧂 Seasoning HTTP connections"
+	connSpinnerID := 0
+	openConnSpinnerID := 0
+	const (
+		httpID = iota + 1
+		openHTTPID
+		sslID
+		openSSLID
+	)
+
+	if importer.ResultCounts.OpenConn > 0 {
+		spinners = append(spinners, progressbar.NewSpinner("Sifting open IP connections...", openConnSpinnerID))
+		connSpinnerID = 1
+		if importer.ResultCounts.OpenSSL > 0 {
+			barList = append(barList, progressbar.NewBar("🧂 Seasoning open SSL connections ", openSSLID, progress.New(gradient)))
+			sslBarName += "     "
+		}
+		if importer.ResultCounts.OpenHTTP > 0 {
+			barList = append(barList, progressbar.NewBar("🧂 Seasoning open HTTP connections", openHTTPID, progress.New(gradient)))
+			httpBarName += "     "
+		}
+	}
+
+	barList = append(barList, progressbar.NewBar(sslBarName, sslID, progress.New(gradient)))
+	barList = append(barList, progressbar.NewBar(httpBarName, httpID, progress.New(gradient)))
+	spinners = append(spinners, progressbar.NewSpinner("Sifting IP connections...", connSpinnerID))
+	bars := progressbar.New(ctx, barList, spinners)
+
+	linkingErrGroup.Go(func() error {
+		_, err := bars.Run()
+		if err != nil {
+			logger.Error().Err(err).Msg("unable to display progress for connection correlation")
+			return fmt.Errorf("unable to display progress for connection correlation: %w", err)
+		}
+		return err
+	})
+
+	linkingErrGroup.Go(func() error {
+		err := importer.writeLinkedHTTP(ctx, bars, httpID, httpWriter, connWriter, false)
+		if err != nil {
+			logger.Error().Err(err).Msg("unable to link http connections")
+		}
+		return err
+	})
+
+	linkingErrGroup.Go(func() error {
+		err := importer.writeLinkedSSL(ctx, bars, sslID, sslWriter, false)
+		if err != nil {
+			logger.Error().Err(err).Msg("unable to link ssl connections")
+		}
+		return err
+	})
+
+	linkingErrGroup.Go(func() error {
+		err := importer.writeUnfilteredConns(bars, false, connSpinnerID)
+		if err != nil {
+			logger.Error().Err(err).Msg("unable to link IP connections")
+		}
+		return err
+	})
+
+	if importer.ResultCounts.OpenConn > 0 {
+		linkingErrGroup.Go(func() error {
+			err := importer.writeUnfilteredConns(bars, true, openConnSpinnerID)
+			if err != nil {
+				logger.Error().Err(err).Msg("unable to link open IP connections")
+			}
+			return err
+		})
+		if importer.ResultCounts.OpenSSL > 0 {
+			linkingErrGroup.Go(func() error {
+				err := importer.writeLinkedSSL(ctx, bars, openSSLID, openSSLWriter, true)
+				if err != nil {
+					logger.Error().Err(err).Msg("unable to link open ssl connections")
+				}
+				return err
+			})
+		}
+		if importer.ResultCounts.OpenHTTP > 0 {
+			linkingErrGroup.Go(func() error {
+				err := importer.writeLinkedHTTP(ctx, bars, openHTTPID, openHTTPWriter, openConnWriter, true)
+				if err != nil {
+					logger.Error().Err(err).Msg("unable to link open http connections")
+				}
+				return err
+			})
+		}
+	}
+
+	if err := linkingErrGroup.Wait(); err != nil {
+		return fmt.Errorf("could not perform connection linking: %w", err)
+	}
+
+	sslWriter.Close()
+	openSSLWriter.Close()
+	httpWriter.Close()
+	openHTTPWriter.Close()
+	connWriter.Close()
+	openConnWriter.Close()
+
+	// // don't truncate tmp tables in debug mode
+	// // these tables should be truncated before each import
+	if zerolog.DebugMode {
+		return nil
+	}
+	return importer.Database.TruncateTmpLinkTables()
+}

importer/parser.go

@@ -0,0 +1,571 @@
+package importer
+
+import (
+	zerolog "activecm/rita/logger"
+	"activecm/rita/util"
+	"bufio"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/spf13/afero"
+)
+
+var errTruncated = errors.New("log file is potentially truncated")
+var errUnknownFileType = errors.New("failed to parse log file: unknown file type or malformed header")
+var errMismatchedPathField = errors.New("TSV 'path' field does not match file pathname prefix")
+
+// ZeekHeader stores vars in the header of the zeek log
+type ZeekHeader[Z zeekRecord] struct {
+	separator             string
+	setSeparator          string
+	emptyField            string
+	unsetField            string
+	path                  string
+	open                  time.Time
+	fieldOrder            []string
+	rawFields             string
+	rawTypes              string
+	isTSV                 bool
+	isJSON                bool
+	headerToStructMapping map[string]int
+	fsPath                string // actual file system path of log
+}
+
+type MetaDBFile struct {
+	importID util.FixedString
+	database string
+	fileHash util.FixedString
+	path     string
+}
+
+// ZeekDateTimeFmt is the common format for zeek header datetimes
+const ZeekDateTimeFmt = "2006-01-02-15-04-05"
+
+const ConnPrefix = "conn"
+const OpenConnPrefix = "open_conn"
+const DNSPrefix = "dns"
+const HTTPPrefix = "http"
+const OpenHTTPPrefix = "open_http"
+const SSLPrefix = "ssl"
+const OpenSSLPrefix = "open_ssl"
+const ConnSummaryPrefixUnderscore = "conn_summary"
+const ConnSummaryPrefixHyphen = "conn-summary"
+
+const lineErrorLimit = 25
+
+// parseFile is a generic function that determines if a passed in path belongs to a tsv or json file, parses the file header and scans through each subsequent line,
+// parsing/unmarshaling it into its associated zeektype and sending it on the passed in generic channel. The generic type is based on the path's prefix in the calling
+// function.
+func parseFile[Z zeekRecord](afs afero.Fs, path string, entryChan chan<- Z, errc chan<- error, metaDBChan chan<- MetaDBFile, database string, importID util.FixedString) {
+	logger := zerolog.GetLogger()
+
+	// open file for reading
+	empty, err := afero.IsEmpty(afs, path)
+	if err != nil {
+		logger.Err(err).Str("path", path).Msg("could not determine if file is empty")
+		return
+	}
+
+	// skip file if it is empty and log a warning
+	if empty {
+		logger.Warn().Str("path", path).Msg("failed to parse log file: file is empty")
+		return
+	}
+
+	file, err := afs.Open(path)
+	if err != nil {
+		logger.Err(err).Str("path", path).Msg("could not open file for parsing")
+		return
+	}
+	defer file.Close()
+
+	fileHash, err := util.NewFixedStringHash(path)
+	if err != nil {
+		logger.Err(err).Str("path", path).Msg("could not hash file path")
+		return
+	}
+
+	metaDBFileEntry := MetaDBFile{
+		importID: importID,
+		database: database,
+		fileHash: fileHash,
+		path:     path,
+	}
+
+	// set up a new scanner to read from file
+	var scanner *bufio.Scanner
+	if strings.HasSuffix(path, ".gz") {
+		// create gzip reader if the file extension insinuates that the file is compressed
+		gzipReader, err := gzip.NewReader(file)
+		if err != nil { // handle error from scanner
+			logger.Err(err).Str("path", path).Msg("failed to parse log file: could not open compressed file")
+			return
+		}
+		scanner = bufio.NewScanner(gzipReader)
+		defer gzipReader.Close()
+	} else {
+		scanner = bufio.NewScanner(file)
+	}
+
+	// set a buffer for the scanner
+	initialBufferSize := 64 * 1024 // 64KiB
+	maxBufferSize := 1024 * 1024   // 1MiB
+	scanner.Buffer(make([]byte, 0, initialBufferSize), maxBufferSize)
+
+	// declare new header object for parsing tsv headers
+	var header ZeekHeader[Z]
+	header.headerToStructMapping = make(map[string]int)
+
+	var typeArr []string
+
+	// declare a generic log entry object
+	var entry Z
+
+	// create line error counter which will allow us to stop scanning in lines from
+	// a file that had more than a certain amount of errors
+	lineErrorCounter := 0
+
+	previousLineHadError := false
+
+	// iterate over lines in file
+	for scanner.Scan() {
+		// handle error from scanner
+		if scanner.Err() != nil {
+			logger.Err(err).Str("path", path).Msg("failed to parse log file: could not scan the file")
+			return
+		}
+
+		// skip empty lines
+		if len(scanner.Bytes()) < 1 {
+			continue
+		}
+
+		// if header type has not been set, attempt to determine log format
+		if !header.isJSON && !header.isTSV {
+
+			switch {
+
+			// Since this line is a comment (it starts with a #), try to parse header in tsv format
+			case scanner.Bytes()[0] == '#':
+				// there are multiple comment lines that make up the header, so we need to call this function
+				// several times until the lines we scan are no longer comments in order to populate the header info
+				typeArr, err = header.parseHeader(scanner.Text())
+
+				// return since parsing of tsv header failed and file is not json
+				if err != nil {
+					logger.Error().Err(err).Str("path", path).Msg("failed to parse log file: unable to parse TSV Zeek header")
+					return
+				}
+
+			// Since the line does not begin with a comment, attempt to check if it is json
+			case scanner.Bytes()[0] == '{' && jsoniter.ConfigCompatibleWithStandardLibrary.Valid(scanner.Bytes()):
+				header.isJSON = true
+				metaDBChan <- metaDBFileEntry
+
+			// Line is not JSON and is not a comment
+			default:
+				// check if tsv header was parsed successfully
+				if header.separator != "" && len(header.fieldOrder) > 0 {
+
+					// set the isTSV header field to true and map the names of the header fields to the struct.
+					header.isTSV = true
+
+					// check & warn if path field doesn't match filename prefix
+					header.fsPath = path
+					err := header.validatePathPrefix()
+					if err != nil {
+						logger.Error().Str("path", path).Err(err).Send()
+					}
+					err = header.mapHeader()
+
+					// return since mapping of tsv header failed and file is not json
+					if err != nil {
+						logger.Err(err).Str("path", path).Msg("failed to parse log file: could not detect valid TSV Zeek header, is file valid TSV or JSON?")
+						return
+					}
+					metaDBChan <- metaDBFileEntry
+
+					// if no header fields were found, quit parsing this file
+				} else {
+					logger.Err(errUnknownFileType).Str("path", path).Send()
+					errc <- errUnknownFileType
+					return
+				}
+			}
+		}
+
+		// parse this line as JSON if we've determined this file is in JSON format
+		if header.isJSON {
+			previousLineHadError = false
+			// unmarshal line
+			if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(scanner.Bytes(), &entry); err != nil {
+				logger.Err(err).Str("path", path).Bytes("record", scanner.Bytes()).Msg("failed to unmarshal line from JSON")
+				lineErrorCounter++
+				previousLineHadError = true
+				if lineErrorCounter > lineErrorLimit {
+					logger.Warn().Str("path", path).Msg("failed to parse log file: file is potentially corrupted")
+					// set this flag to false so that we don't log that this file could be truncated
+					previousLineHadError = false
+					break
+				}
+				continue
+			}
+
+			// set log path field
+			data := reflect.ValueOf(&entry).Elem()
+			data.FieldByName("LogPath").SetString(path)
+
+			// send parsed entry to its appropriate channel
+			entryChan <- entry
+
+			resetZeekRecord(&entry)
+
+			// parse this line as TSV if we've determined this file is in TSV format
+		} else if header.isTSV {
+			previousLineHadError = false
+
+			// don't parse this line if it is a comment
+			if scanner.Bytes()[0] == '#' {
+				continue
+			}
+			// get the type of zeek log record this entry is
+			data := reflect.ValueOf(&entry).Elem()
+
+			// reset the entry just to be safe
+			data.Set(reflect.Zero(data.Type()))
+
+			// scan in line
+			line := scanner.Text()
+
+			// track whether or not this line had an error when parsing any fields
+			lineHadError := false
+
+			// set the end index of the field itself to the index of the next tab (or separator)
+			fieldEndIndex := strings.Index(line, header.separator)
+
+			// set field counter
+			idx := 0
+
+			// loop through all but last fields in line
+			for fieldEndIndex > -1 && idx < len(header.fieldOrder) {
+
+				// check if the header field is in the struct
+				if header.headerToStructMapping[header.fieldOrder[idx]] > -1 {
+					// parse field if not empty or unset
+					if line[:fieldEndIndex] != header.emptyField && line[:fieldEndIndex] != header.unsetField {
+
+						// parse field by assigning the correlating struct field using reflection
+						err := header.parseField(
+							line[:fieldEndIndex], // the field itself, sliced out of the line
+							typeArr[idx],         // the zeek type of the field
+							data.Field(header.headerToStructMapping[header.fieldOrder[idx]])) // the struct field to update
+
+						if err != nil {
+							logger.Warn().Err(err).
+								Str("path", path).
+								Str("field_name", header.fieldOrder[idx]).
+								Str("field_value", line).
+								Msg("failed to parse field in TSV Zeek log")
+							lineHadError = true
+							previousLineHadError = true
+						}
+					}
+
+				}
+				// reslice line to first of next field to the end of the line
+				line = line[fieldEndIndex+len(header.separator):]
+
+				// update the end index of the field to the index of the next tab (or separator)
+				fieldEndIndex = strings.Index(line, header.separator)
+				idx++
+			}
+
+			if fieldEndIndex == -1 && idx < len(header.fieldOrder)-2 {
+				logger.Err(errTruncated).Str("path", path).Send()
+				errc <- errTruncated
+				break
+			}
+
+			// parse in last field
+			if idx < len(header.fieldOrder) && line != header.emptyField && line != header.unsetField &&
+				header.headerToStructMapping[header.fieldOrder[idx]] > -1 {
+				err := header.parseField(
+					line,         // the last field, now the only thing left in line
+					typeArr[idx], // the zeek type of the field
+					data.Field(header.headerToStructMapping[header.fieldOrder[idx]])) // the struct field to update
+
+				if err != nil {
+					logger.Warn().Err(err).
+						Str("path", path).
+						Str("field_name", header.fieldOrder[idx]).
+						Str("field_value", line).
+						Msg("failed to parse field in TSV Zeek log")
+					lineHadError = true
+					previousLineHadError = true
+				}
+			}
+
+			// increment file parsing error count if there were errors during field parsing
+			if lineHadError {
+				lineErrorCounter++
+			}
+
+			// return if parsing error limit for file was reached
+			if lineErrorCounter > lineErrorLimit {
+				logger.Warn().Str("path", path).Msg("log file is potentially corrupted")
+				// set this flag to false so that we don't log that this file could be truncated
+				previousLineHadError = false
+				break
+			}
+
+			// set log path field
+			data.FieldByName("LogPath").SetString(path)
+
+			// send parsed entry to its appropriate channel
+			entryChan <- entry
+
+			// reset the zeek record entry just in case
+			resetZeekRecord(&entry)
+		}
+	}
+
+	// if last line of log had an error, indicate that file may be truncated
+	if previousLineHadError {
+		logger.Err(errTruncated).Str("path", path).Send()
+		errc <- errTruncated
+	}
+}
+
+// parseHeader parses the header of a Zeek log in TSV format
+func (header *ZeekHeader[Z]) parseHeader(line string) (typeArr []string, err error) {
+
+	potentialFields := strings.Fields(line)
+	// 	grabs from the comment # to the space to get the first field value
+	potentialFieldName := potentialFields[0][1:]
+	potentialFieldValue := convertHexFieldValue(potentialFields[1])
+
+	switch potentialFieldName {
+	case "separator":
+		header.separator = potentialFieldValue
+	case "set_separator":
+		header.setSeparator = potentialFieldValue
+	case "unset_field":
+		header.unsetField = potentialFieldValue
+	case "path":
+		header.path = potentialFieldValue
+	case "empty_field":
+		header.emptyField = potentialFieldValue
+	case "open":
+		var dateParseErr error
+		header.open, dateParseErr = time.Parse(ZeekDateTimeFmt, potentialFieldValue)
+		if dateParseErr != nil {
+			return nil, fmt.Errorf("date not parsed for open field: %v", dateParseErr.Error())
+		}
+	case "fields":
+		header.rawFields = line
+	case "types":
+		header.rawTypes = line
+	}
+	// 	map zeek fields and types, get field order
+	if len(header.rawFields) > 0 && len(header.rawTypes) > 0 {
+		splitFields := strings.Fields(header.rawFields)
+		splitTypes := strings.Fields(header.rawTypes)
+
+		splitFields = splitFields[1:]
+		splitTypes = splitTypes[1:]
+
+		if len(splitTypes) == len(splitFields) {
+			typeArr = make([]string, len(splitFields))
+			for idx := range splitFields {
+				// track the field names by the order they appear in the header
+				header.fieldOrder = append(header.fieldOrder, splitFields[idx])
+				// track the field types by the order they appear in the header
+				typeArr[idx] = splitTypes[idx]
+			}
+			return typeArr, nil
+		}
+
+		return nil, fmt.Errorf("mismatched header fields. zeek types: %v, zeek fields: %v", splitTypes, splitFields)
+	}
+
+	return typeArr, nil
+}
+
+// mapHeader maps the names of the fields found in the log header to the corresponding
+// struct field's "index". This allows the struct to be dynamically populated using reflection.
+func (header *ZeekHeader[Z]) mapHeader() error {
+	// creates an empty object of the generic type so that reflect can determine which
+	// log type we are dealing with
+	var entry Z
+	structType := reflect.TypeOf(entry)
+
+	// walk the fields of the zeekData, making sure the zeekData struct has
+	// an equal number of named zeek fields and zeek types
+	for i := 0; i < structType.NumField(); i++ {
+		structField := structType.Field(i)
+		zeekName := structField.Tag.Get("zeek")
+		zeekType := structField.Tag.Get("zeektype")
+
+		// If this field is not associated with zeek, skip it
+		if len(zeekName) == 0 && len(zeekType) == 0 {
+			continue
+		}
+
+		if len(zeekName) == 0 || len(zeekType) == 0 {
+			return errors.New("invalid zeek field")
+		}
+
+		header.headerToStructMapping[zeekName] = i
+
+	}
+
+	// Make sure that fields that are in the header and not in the struct definition get ignored
+	// walks the fields of the header and sets the mapping for any header fields that are not
+	// in the struct to a -1, otherwise looking up the map will return a 0 which will break parsing
+	for _, headerName := range header.fieldOrder {
+		if _, ok := header.headerToStructMapping[headerName]; !ok {
+			header.headerToStructMapping[headerName] = -1
+		}
+	}
+
+	return nil
+}
+
+// parseField parses a single field in a zeek log record
+func (header *ZeekHeader[Z]) parseField(value string, zeekType string, resultField reflect.Value) error {
+	// handle data cleaning / conversion for the different zeek types
+	switch zeekType {
+	case "time":
+		decimalPointIdx := strings.Index(value, ".")
+		if decimalPointIdx == -1 {
+			return fmt.Errorf("couldn't convert unix ts: no decimal point in timestamp: %v", value)
+		}
+
+		s, err := strconv.Atoi(value[:decimalPointIdx])
+		if err != nil {
+			return fmt.Errorf("couldn't convert unix ts: %v", err.Error())
+		}
+
+		nanos, err := strconv.Atoi(value[decimalPointIdx+1:])
+		if err != nil {
+			return fmt.Errorf("couldn't convert unix ts: %v", err.Error())
+		}
+
+		ttim := time.Unix(int64(s), int64(nanos))
+		tval := ttim.Unix()
+		resultField.SetInt(tval)
+	case "interval":
+		intervalFloat, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+		if err != nil {
+			return fmt.Errorf("couldn't convert zeektype interval: %v", err.Error())
+		}
+		tval := reflect.ValueOf(intervalFloat)
+		resultField.Set(tval)
+	case "string":
+		fallthrough
+	case "enum":
+		fallthrough
+	case "addr":
+		resultField.SetString(value)
+	case "count":
+		countInt, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
+		if err != nil {
+			return fmt.Errorf("couldn't convert zeektype count: %v", err.Error())
+		}
+		resultField.SetInt(countInt)
+	case "port":
+		portInt, err := strconv.Atoi(strings.TrimSpace(value))
+		if err != nil {
+			return fmt.Errorf("couldn't convert zeektype port: %v", err.Error())
+		}
+		resultField.SetInt(int64(portInt))
+	case "bool":
+		boolCvt, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("couldn't convert zeektype bool: %v", err.Error())
+		}
+		resultField.SetBool(boolCvt)
+	case "set[string]":
+		fallthrough
+	case "set[enum]":
+		fallthrough
+	case "vector[string]":
+		strsSplit := strings.Split(value, header.setSeparator)
+		tval := reflect.ValueOf(strsSplit)
+		resultField.Set(tval)
+	case "vector[interval]":
+		var intervals []float64
+		strNums := strings.Split(value, header.setSeparator)
+		for _, str := range strNums {
+			intervalFloat, err := strconv.ParseFloat(strings.TrimSpace(str), 64)
+			if err != nil {
+				return fmt.Errorf("couldn't convert zeektype: vector[interval] %w", err)
+			}
+			intervals = append(intervals, intervalFloat)
+		}
+		tval := reflect.ValueOf(intervals)
+		resultField.Set(tval)
+	default:
+	}
+
+	return nil
+}
+
+// validatePathPrefix returns an error if the TSV header path field does not match the prefix of the file's path name
+func (header *ZeekHeader[Z]) validatePathPrefix() (err error) {
+	switch {
+	case strings.HasPrefix(filepath.Base(header.fsPath), ConnPrefix) && !strings.HasPrefix(filepath.Base(header.fsPath), ConnSummaryPrefixUnderscore) && !strings.HasPrefix(filepath.Base(header.fsPath), ConnSummaryPrefixHyphen):
+		if header.path != ConnPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), OpenConnPrefix):
+		if header.path != OpenConnPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), DNSPrefix):
+		if header.path != DNSPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), HTTPPrefix):
+		if header.path != HTTPPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), OpenHTTPPrefix):
+		if header.path != OpenHTTPPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), SSLPrefix):
+		if header.path != SSLPrefix {
+			return errMismatchedPathField
+		}
+	case strings.HasPrefix(filepath.Base(header.fsPath), OpenSSLPrefix):
+		if header.path != OpenSSLPrefix {
+			return errMismatchedPathField
+		}
+	}
+	return nil
+}
+
+// convertHexFieldValue converts any hex encoded zeek field values to normal characters
+// if err is true, conversion was not needed and original value is returned
+// ie: tab char = \x09
+func convertHexFieldValue(givenValue string) string {
+	newValue, err := strconv.Unquote("\"" + givenValue + "\"")
+	if err != nil {
+		return givenValue
+	}
+	return newValue
+}
+
+// resetZeekRecord resets the zeek record with values that represent zero
+func resetZeekRecord(r any) {
+	p := reflect.ValueOf(r).Elem()
+	p.Set(reflect.Zero(p.Type()))
+}

importer/parser_test.go

@@ -0,0 +1,248 @@
+package importer
+
+import (
+	"activecm/rita/importer/zeektypes"
+	"activecm/rita/util"
+	"errors"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTruncatedTSV(t *testing.T) {
+	path := "../test_data/truncated/conn.log"
+	entries := make(chan zeektypes.Conn)
+	errc := make(chan error)
+	metaDBChan := make(chan MetaDBFile)
+
+	// get the current time in microseconds
+	start := time.Now().UTC().UnixMicro()
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(start, 10))
+	require.NoError(t, err)
+
+	go func() {
+		parseFile(afero.NewOsFs(), path, entries, errc, metaDBChan, "test", importID)
+		close(errc)
+		close(entries)
+		close(metaDBChan)
+	}()
+
+	receivedTruncatedErr := false
+	openChannels := 3
+	for openChannels > 0 {
+		select {
+		case _, ok := <-entries:
+			if !ok {
+				openChannels--
+			}
+		case _, ok := <-metaDBChan:
+			if !ok {
+				openChannels--
+			}
+		case err, ok := <-errc:
+			if !ok {
+				openChannels--
+			} else if errors.Is(err, errTruncated) {
+				receivedTruncatedErr = true
+			}
+
+		}
+	}
+
+	require.True(t, receivedTruncatedErr)
+
+}
+
+func TestTruncatedHeader(t *testing.T) {
+	path := "../test_data/truncated/conn_top.log"
+	entries := make(chan zeektypes.Conn)
+	errc := make(chan error)
+	metaDBChan := make(chan MetaDBFile)
+
+	// get the current time in microseconds
+	start := time.Now().UTC().UnixMicro()
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(start, 10))
+	require.NoError(t, err)
+
+	go func() {
+		parseFile(afero.NewOsFs(), path, entries, errc, metaDBChan, "test", importID)
+		close(errc)
+		close(entries)
+		close(metaDBChan)
+	}()
+
+	receivedUnknownFileErr := false
+	openChannels := 3
+	for openChannels > 0 {
+		select {
+		case _, ok := <-entries:
+			if !ok {
+				openChannels--
+			}
+		case _, ok := <-metaDBChan:
+			if !ok {
+				openChannels--
+			}
+		case err, ok := <-errc:
+			if !ok {
+				openChannels--
+			} else if errors.Is(err, errUnknownFileType) {
+				receivedUnknownFileErr = true
+			}
+		}
+	}
+
+	require.True(t, receivedUnknownFileErr)
+}
+
+func TestTruncatedJSON(t *testing.T) {
+	path := "../test_data/truncated/conn_json.log"
+	entries := make(chan zeektypes.Conn)
+	errc := make(chan error)
+	metaDBChan := make(chan MetaDBFile)
+
+	// get the current time in microseconds
+	start := time.Now().UTC().UnixMicro()
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(start, 10))
+	require.NoError(t, err)
+
+	go func() {
+		parseFile(afero.NewOsFs(), path, entries, errc, metaDBChan, "test", importID)
+		close(errc)
+		close(entries)
+		close(metaDBChan)
+	}()
+
+	receivedTruncatedErr := false
+	openChannels := 3
+	for openChannels > 0 {
+		select {
+		case _, ok := <-entries:
+			if !ok {
+				openChannels--
+			}
+		case _, ok := <-metaDBChan:
+			if !ok {
+				openChannels--
+			}
+		case err, ok := <-errc:
+			if !ok {
+				openChannels--
+			} else if errors.Is(err, errTruncated) {
+				receivedTruncatedErr = true
+			}
+
+		}
+	}
+
+	require.True(t, receivedTruncatedErr)
+
+}
+
+func TestHasUnknownFieldTSV(t *testing.T) {
+	path := "../test_data/has_unknown_field/http.log"
+
+	entries := make(chan zeektypes.HTTP)
+	errc := make(chan error)
+	metaDBChan := make(chan MetaDBFile)
+
+	// get the current time in microseconds
+	start := time.Now().UTC().UnixMicro()
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(start, 10))
+	require.NoError(t, err)
+
+	go func() {
+		parseFile(afero.NewOsFs(), path, entries, errc, metaDBChan, "test", importID)
+		close(errc)
+		close(entries)
+		close(metaDBChan)
+	}()
+
+	receivedErr := false
+	openChannels := 3
+	recordCount := 0
+	for openChannels > 0 {
+		select {
+		case _, ok := <-entries:
+			if !ok {
+				openChannels--
+			} else {
+				recordCount++
+			}
+		case _, ok := <-metaDBChan:
+			if !ok {
+				openChannels--
+			}
+		case err, ok := <-errc:
+			if !ok {
+				openChannels--
+			} else if err != nil {
+				receivedErr = true
+			}
+
+		}
+	}
+
+	require.Equal(t, 5, recordCount, "number of http records")
+	require.False(t, receivedErr, "unknown field error")
+
+}
+
+func TestPlainTextFile(t *testing.T) {
+	path := "../test_data/text_file/conn.log"
+
+	entries := make(chan zeektypes.Conn)
+	errc := make(chan error)
+	metaDBChan := make(chan MetaDBFile)
+
+	// get the current time in microseconds
+	start := time.Now().UTC().UnixMicro()
+
+	// create a unique import id using the start time
+	importID, err := util.NewFixedStringHash(strconv.FormatInt(start, 10))
+	require.NoError(t, err)
+
+	go func() {
+		parseFile(afero.NewOsFs(), path, entries, errc, metaDBChan, "test", importID)
+		close(errc)
+		close(entries)
+		close(metaDBChan)
+	}()
+
+	receivedErr := false
+	openChannels := 3
+	recordCount := 0
+	for openChannels > 0 {
+		select {
+		case _, ok := <-entries:
+			if !ok {
+				openChannels--
+			} else {
+				recordCount++
+			}
+		case _, ok := <-metaDBChan:
+			if !ok {
+				openChannels--
+			}
+		case err, ok := <-errc:
+			if !ok {
+				openChannels--
+			} else if errors.Is(err, errUnknownFileType) {
+				receivedErr = true
+			}
+
+		}
+	}
+	require.True(t, receivedErr, "should receive unknown file type error")
+}

importer/ssl.go

@@ -0,0 +1,271 @@
+package importer
+
+import (
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/importer/zeektypes"
+	"activecm/rita/logger"
+	"activecm/rita/progressbar"
+	"activecm/rita/util"
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/google/uuid"
+)
+
+var errServerNameEmpty = errors.New("server name is blank")
+
+type SSLEntry struct {
+	ImportTime       time.Time        `ch:"import_time"`
+	ZeekUID          util.FixedString `ch:"zeek_uid"`
+	Hash             util.FixedString `ch:"hash"`
+	Timestamp        time.Time        `ch:"ts"`
+	Src              net.IP           `ch:"src"`
+	Dst              net.IP           `ch:"dst"`
+	SrcNUID          uuid.UUID        `ch:"src_nuid"`
+	DstNUID          uuid.UUID        `ch:"dst_nuid"`
+	SrcPort          uint16           `ch:"src_port"`
+	DstPort          uint16           `ch:"dst_port"`
+	Duration         float64          `ch:"duration"`
+	SrcLocal         bool             `ch:"src_local"`
+	DstLocal         bool             `ch:"dst_local"`
+	SrcBytes         int64            `ch:"src_bytes"`
+	DstBytes         int64            `ch:"dst_bytes"`
+	SrcIPBytes       int64            `ch:"src_ip_bytes"`
+	DstIPBytes       int64            `ch:"dst_ip_bytes"`
+	SrcPackets       int64            `ch:"src_packets"`
+	DstPackets       int64            `ch:"dst_packets"`
+	Proto            string           `ch:"proto"`
+	Service          string           `ch:"service"`
+	ConnState        string           `ch:"conn_state"`
+	Version          string           `ch:"version"`
+	Cipher           string           `ch:"cipher"`
+	Curve            string           `ch:"curve"`
+	ServerName       string           `ch:"server_name"`
+	Resumed          bool             `ch:"resumed"`
+	NextProtocol     string           `ch:"next_protocol"`
+	Established      bool             `ch:"established"`
+	ServerCertFUIDs  []string         `ch:"server_cert_fuids"`
+	ClientCertFUIDs  []string         `ch:"client_cert_fuids"`
+	ServerSubject    string           `ch:"server_subject"`
+	ServerIssuer     string           `ch:"server_issuer"`
+	ClientSubject    string           `ch:"client_subject"`
+	ClientIssuer     string           `ch:"client_issuer"`
+	ValidationStatus string           `ch:"validation_status"`
+	JA3              string           `ch:"ja3"`
+	JA3S             string           `ch:"ja3s"`
+}
+
+// parseSSL listens on a channel of raw ssl/openssl log records, formats them and sends them to be linked with conn/openconn records and written to the database
+// func parseSSL(ssl <-chan zeektypes.SSL, zeekUIDMap cmap.ConcurrentMap[string, *ZeekUIDRecord], uSSLMap cmap.ConcurrentMap[string, *UniqueFQDN], output chan database.Data, numSSL *uint64) {
+func parseSSL(ssl <-chan zeektypes.SSL, output chan database.Data, importTime time.Time, numSSL *uint64) {
+	// logger := zlog.GetLogger()
+
+	// loop over raw ssl/openssl channel
+	for s := range ssl {
+
+		// parse raw record record as an ssl/openssl entry
+		entry, err := formatSSLRecord(&s, importTime)
+		if err != nil {
+			// logger.Warn().Err(err).
+			// 	Str("log_path", s.LogPath).
+			// 	Str("zeek_uid", s.UID).
+			// 	Str("timestamp", (time.Unix(int64(s.TimeStamp), 0)).String()).
+			// 	Str("src", s.Source).
+			// 	Str("dst", s.Destination).
+			// 	Str("sni", s.ServerName).
+			// 	Send()
+			continue
+		}
+
+		// entry was subject to filtering
+		if entry == nil {
+			continue
+		}
+
+		output <- entry
+		// increment record counter
+		atomic.AddUint64(numSSL, 1)
+	}
+}
+
+// formatSSLRecord takes a raw ssl record and formats it into the structure needed by the database
+func formatSSLRecord(parseSSL *zeektypes.SSL, importTime time.Time) (*SSLEntry, error) {
+	// logger := zerolog.GetLogger()
+	cfg, err := config.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	// get source destination pair
+	src := parseSSL.Source
+	dst := parseSSL.Destination
+
+	// parse source and destination
+	srcIP := net.ParseIP(src)
+	dstIP := net.ParseIP(dst)
+
+	// verify that both addresses were parsed successfully
+	if (srcIP == nil) || (dstIP == nil) {
+		return nil, errors.New(errParseSrcDst)
+	}
+
+	// get sni
+	sni := parseSSL.ServerName
+
+	if sni == "" {
+		// logger.Debug().
+		// 	Str("log_path", parseSSL.LogPath).
+		// 	Str("zeek_uid", parseSSL.UID).
+		// 	Str("timestamp", (time.Unix(int64(parseSSL.TimeStamp), 0)).String()).
+		// 	Str("src", parseSSL.Source).
+		// 	Str("dst", parseSSL.Destination).
+		// 	Str("sni", parseSSL.ServerName).
+		// 	Msg("sni field is empty")
+		return nil, fmt.Errorf("could not parse SSL connection %s -> %s: %w", src, dst, errServerNameEmpty)
+	}
+
+	ignore := cfg.Filter.FilterDomain(sni) || cfg.Filter.FilterConnPair(srcIP, dstIP) || cfg.Filter.FilterSNIPair(srcIP)
+	if ignore {
+		return nil, nil
+	}
+
+	srcNUID := util.ParseNetworkID(srcIP, parseSSL.AgentUUID)
+	dstNUID := util.ParseNetworkID(dstIP, parseSSL.AgentUUID)
+
+	zeekUID, err := util.NewFixedStringHash(parseSSL.UID)
+	if err != nil {
+		return nil, err
+	}
+
+	hash, err := util.NewFixedStringHash(srcIP.To16().String(), srcNUID.String(), dstIP.To16().String(), dstNUID.String(), sni)
+	if err != nil {
+		return nil, err
+	}
+
+	entry := &SSLEntry{
+		ImportTime:       importTime,
+		ZeekUID:          zeekUID,
+		Hash:             hash,
+		Timestamp:        time.Unix(int64(parseSSL.TimeStamp), 0),
+		Src:              srcIP,
+		Dst:              dstIP,
+		SrcNUID:          srcNUID,
+		DstNUID:          dstNUID,
+		SrcPort:          uint16(parseSSL.SourcePort),
+		DstPort:          uint16(parseSSL.DestinationPort),
+		SrcLocal:         cfg.Filter.CheckIfInternal(srcIP),
+		DstLocal:         cfg.Filter.CheckIfInternal(dstIP),
+		Version:          parseSSL.Version,
+		Cipher:           parseSSL.Cipher,
+		Curve:            parseSSL.Curve,
+		ServerName:       parseSSL.ServerName,
+		Resumed:          parseSSL.Resumed,
+		NextProtocol:     parseSSL.NextProtocol,
+		Established:      parseSSL.Established,
+		ServerCertFUIDs:  parseSSL.CertChainFuids,
+		ClientCertFUIDs:  parseSSL.ClientCertChainFuids,
+		ServerSubject:    parseSSL.Subject,
+		ServerIssuer:     parseSSL.Issuer,
+		ClientSubject:    parseSSL.ClientSubject,
+		ClientIssuer:     parseSSL.ClientIssuer,
+		ValidationStatus: parseSSL.ValidationStatus,
+		JA3:              parseSSL.JA3,
+		JA3S:             parseSSL.JA3S,
+	}
+
+	return entry, nil
+}
+
+func (importer *Importer) writeLinkedSSL(ctx context.Context, progress *tea.Program, barID int, sslWriter *database.BulkWriter, open bool) error { //httpWriter chan database.Data, connWriter chan database.Data
+	logger := logger.GetLogger()
+
+	var totalSSL uint64
+	err := importer.Database.Conn.QueryRow(importer.Database.GetContext(), `
+		SELECT count() FROM ssl_tmp
+	`).Scan(&totalSSL)
+	if err != nil {
+		return err
+	}
+
+	tmpTable := "ssl_tmp"
+	tableB := "conn_tmp"
+	if open {
+		tmpTable = "openssl_tmp"
+		tableB = "openconn_tmp"
+	}
+
+	chCtx := importer.Database.QueryParameters(clickhouse.Parameters{
+		"tmp_table": tmpTable,
+		"table_b":   tableB,
+	})
+
+	rows, err := importer.Database.Conn.Query(chCtx, `
+	SELECT 
+		s.zeek_uid as zeek_uid, c.ts AS ts, s.src as src, s.src_nuid as src_nuid, s.dst as dst, s.dst_nuid as dst_nuid,
+		s.src_port as src_port, s.dst_port as dst_port, s.src_local as src_local, s.dst_local as dst_local, server_name as server_name,
+		s.version as version, s.cipher as cipher, s.curve as curve, s.resumed as resumed, s.next_protocol as next_protocol, s.established as established, 
+		s.server_cert_fuids as server_cert_fuids, client_cert_fuids, server_subject, server_issuer, client_subject, client_issuer, validation_status,
+		ja3, ja3s,
+		-- set proto and service regardless of whether it was linked already or not
+		-- since multi-requests can use different dst ports and still have the same UID, so
+		-- it is useful to be able to see the dst ports coming from multi request entries as well
+		c.proto as proto, c.service as service,
+		c.src_ip_bytes as src_ip_bytes,
+		c.dst_ip_bytes as dst_ip_bytes,
+		c.src_bytes as src_bytes,
+		c.dst_bytes as dst_bytes,
+		c.duration as duration,
+		c.conn_state as conn_state,
+		c.src_packets as src_packets,
+		c.dst_packets as dst_packets
+	FROM {tmp_table:Identifier} s
+	INNER JOIN {table_b:Identifier} c USING zeek_uid
+    
+`)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	i := 0
+	for rows.Next() {
+		select {
+		// abort this function if the context was cancelled
+		case <-ctx.Done():
+			logger.Warn().Msg("cancelling SSL connection linking")
+			rows.Close()
+			return ctx.Err()
+		default:
+			var entry SSLEntry
+
+			err := rows.ScanStruct(&entry)
+			if err != nil {
+				log.Panicln(err)
+			}
+			i++
+			if i%1000 == 0 {
+				progress.Send(progressbar.ProgressMsg{ID: barID, Percent: float64(float64(i) / float64(totalSSL))})
+			}
+			entry.ImportTime = importer.Database.ImportStartedAt
+
+			hash, err := util.NewFixedStringHash(entry.Src.To16().String(), entry.SrcNUID.String(), entry.ServerName)
+			if err != nil {
+				log.Panicln(err)
+			}
+
+			entry.Hash = hash
+			sslWriter.WriteChannel <- &entry
+		}
+	}
+	rows.Close()
+	progress.Send(progressbar.ProgressMsg{ID: barID, Percent: 1})
+
+	return nil
+}

importer/zeektypes/conn.go

@@ -0,0 +1,139 @@
+package zeektypes
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// EntryTypeConn should be matched against zeekFile.EntryType()
+// before using OpenZeekReader[ZeekConn](fs, zeekFile) to read from the file.
+const EntryTypeConn = "conn"
+
+// EntryTypeOpenConn should be matched against zeekFile.EntryType()
+// before using OpenZeekReader[ZeekConn](fs, zeekFile) to read from the file.
+// Zeek logs written to the open_conn file follow the same format normal conn logs.
+const EntryTypeOpenConn = "open_conn"
+
+// type Timestamp time.Time
+type Timestamp int64
+
+var ErrInvalidZeekTimestamp = errors.New("invalid zeek timestamp")
+
+// Conn provides a data structure for zeek's connection data
+type Conn struct {
+	// TimeStamp of this connection
+	TimeStamp Timestamp `zeek:"ts" zeektype:"time" json:"ts"`
+	// UID is the Unique Id for this connection (generated by zeek)
+	UID string `zeek:"uid" zeektype:"string" json:"uid"`
+	// Source is the source address for this connection
+	Source string `zeek:"id.orig_h" zeektype:"addr" json:"id.orig_h"`
+	// SourcePort is the source port of this connection
+	SourcePort int `zeek:"id.orig_p" zeektype:"port" json:"id.orig_p"`
+	// Destination is the destination of the connection
+	Destination string `zeek:"id.resp_h" zeektype:"addr" json:"id.resp_h"`
+	// DestinationPort is the port at the destination host
+	DestinationPort int `zeek:"id.resp_p" zeektype:"port" json:"id.resp_p"`
+	// Proto is the string protocol identifier for this connection
+	Proto string `zeek:"proto" zeektype:"enum" json:"proto"`
+	// Service describes the service of this connection if there was one
+	Service string `zeek:"service" zeektype:"string" json:"service"`
+	// Duration is the floating point representation of connection length
+	Duration float64 `zeek:"duration" zeektype:"interval" json:"duration"`
+	// OrigBytes is the byte count coming from the origin
+	OrigBytes int64 `zeek:"orig_bytes" zeektype:"count" json:"orig_bytes"`
+	// RespBytes is the byte count coming in on response
+	RespBytes int64 `zeek:"resp_bytes" zeektype:"count" json:"resp_bytes"`
+	// ConnState has data describing the state of a connection
+	ConnState string `zeek:"conn_state" zeektype:"string" json:"conn_state"`
+	// LocalOrigin denotes that the connection originated locally
+	LocalOrigin bool `zeek:"local_orig" zeektype:"bool" json:"local_orig"`
+	// LocalResponse denote that the connection responded locally
+	LocalResponse bool `zeek:"local_resp" zeektype:"bool" json:"local_resp"`
+	// MissedBytes keeps a count of bytes missed
+	MissedBytes int64 `zeek:"missed_bytes" zeektype:"count" json:"missed_bytes"`
+	// History is a string containing historical information
+	History string `zeek:"history" zeektype:"string" json:"history"`
+	// OrigPkts is a count of origin packets
+	OrigPackets int64 `zeek:"orig_pkts" zeektype:"count" json:"orig_pkts"`
+	// OrigIpBytes is another origin data count
+	OrigIPBytes int64 `zeek:"orig_ip_bytes" zeektype:"count" json:"orig_ip_bytes"`
+	// RespPackets counts response packets
+	RespPackets int64 `zeek:"resp_pkts" zeektype:"count" json:"resp_pkts"`
+	// RespIpBytes gives the bytecount of response data
+	RespIPBytes int64 `zeek:"resp_ip_bytes" zeektype:"count" json:"resp_ip_bytes"`
+	// TunnelParents lists tunnel parents
+	TunnelParents []string `zeek:"tunnel_parents" zeektype:"set[string]" json:"tunnel_parents"`
+	// AgentHostname names which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentHostname string `zeek:"agent_hostname" zeektype:"string" json:"agent_hostname"`
+	// AgentUUID identifies which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentUUID string `zeek:"agent_uuid" zeektype:"string" json:"agent_uuid"`
+	// Path of log file containing this record
+	LogPath string
+}
+
+func (c *Conn) SetLogPath(path string) { c.LogPath = path }
+
+// Unmarshals JSON timestamps
+func (ts *Timestamp) UnmarshalJSON(data []byte) error {
+	var t interface{}
+	if err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(data, &t); err != nil {
+		return err
+	}
+
+	switch input := t.(type) {
+	// all number types are assumed to be in unix format, possibly with fractional seconds
+	case int:
+		tsVal, ok := t.(Timestamp)
+		if !ok {
+			return ErrInvalidZeekTimestamp
+		}
+		*ts = tsVal
+	case int32:
+		tsVal, ok := t.(Timestamp)
+		if !ok {
+			return ErrInvalidZeekTimestamp
+		}
+		*ts = tsVal
+	case float32:
+		tsVal, ok := t.(Timestamp)
+		if !ok {
+			return ErrInvalidZeekTimestamp
+		}
+		*ts = tsVal
+	case int64:
+		intVal, ok := t.(int64)
+		if !ok {
+			return ErrInvalidZeekTimestamp
+		}
+		*ts = Timestamp(intVal)
+	case float64:
+		floatVal, ok := t.(float64)
+		if !ok {
+			return ErrInvalidZeekTimestamp
+		}
+		*ts = Timestamp(floatVal)
+	case string:
+		// assumed to be in RFC8601 format, though other formats can be added as necessary
+		// ex: 2019-11-13T09:00:01.932360Z
+		t, err := time.Parse(time.RFC3339, input)
+		if err != nil {
+			// attempt to parse as a epoch
+			tsVal, err := strconv.ParseFloat(strings.TrimSpace(input), 64)
+			if err != nil {
+				return errors.Join(ErrInvalidZeekTimestamp, err)
+			}
+			*ts = Timestamp(tsVal)
+		}
+		var unix Timestamp
+		unix = Timestamp(t.UTC().Unix())
+		*ts = unix
+	default:
+		return ErrInvalidZeekTimestamp
+	}
+
+	return nil
+}

importer/zeektypes/dns.go

@@ -0,0 +1,65 @@
+package zeektypes
+
+// EntryTypeDNS should be matched against zeekFile.EntryType()
+// before using OpenZeekReader[ZeekDNS](fs, zeekFile) to read from the file.
+const EntryTypeDNS = "dns"
+
+// DNS provides a data structure for entries in the zeek DNS log
+type DNS struct {
+	// TimeStamp of this connection
+	TimeStamp Timestamp `zeek:"ts" zeektype:"time" json:"ts"`
+	// UID is the Unique Id for this connection (generated by zeek)
+	UID string `zeek:"uid" zeektype:"string" json:"uid"`
+	// Source is the source address for this connection
+	Source string `zeek:"id.orig_h" zeektype:"addr" json:"id.orig_h"`
+	// SourcePort is the source port of this connection
+	SourcePort int `zeek:"id.orig_p" zeektype:"port" json:"id.orig_p"`
+	// Destination is the destination of the connection
+	Destination string `zeek:"id.resp_h" zeektype:"addr" json:"id.resp_h"`
+	// DestinationPort is the port at the destination host
+	DestinationPort int `zeek:"id.resp_p" zeektype:"port" json:"id.resp_p"`
+	// Proto is the string protocol identifier for this connection
+	Proto string `zeek:"proto" zeektype:"enum" json:"proto"`
+	// TransID contains a 16 bit identifier assigned by the program that generated the query
+	TransID int64 `zeek:"trans_id" zeektype:"count" json:"trans_id"`
+	// RTT contains the round trip time of this request / response
+	RTT float64 `zeek:"rtt" zeektype:"interval" json:"rtt"`
+	// Query contains the query string
+	Query string `zeek:"query" zeektype:"string" json:"query"`
+	// QClass contains a the qclass of the query
+	QClass int64 `zeek:"qclass" zeektype:"count" json:"qclass"`
+	// QClassName contains a descriptive name for the query
+	QClassName string `zeek:"qclass_name" zeektype:"string" json:"qclass_name"`
+	// QType contains the value of the query type
+	QType int64 `zeek:"qtype" zeektype:"count" json:"qtype"`
+	// QTypeName provides a descriptive name for the query
+	QTypeName string `zeek:"qtype_name" zeektype:"string" json:"qtype_name"`
+	// RCode contains the response code value from the DNS messages
+	RCode int64 `zeek:"rcode" zeektype:"count" json:"rcode"`
+	// RCodeName provides a descriptive name for RCode
+	RCodeName string `zeek:"rcode_name" zeektype:"string" json:"rcode_name"`
+	// AA represents the state of the authoritive answer bit of the resp messages
+	AA bool `bson:"AA" zeek:"AA" zeektype:"bool" json:"AA"`
+	// TC represents the truncation bit of the message
+	TC bool `bson:"TC" zeek:"TC" zeektype:"bool" json:"TC"`
+	// RD represens the recursion desired bit of the message
+	RD bool `bson:"RD" zeek:"RD" zeektype:"bool" json:"RD"`
+	// RA represents the recursion available bit of the message
+	RA bool `bson:"RA" zeek:"RA" zeektype:"bool" json:"RA"`
+	// Z represents the state of a reseverd field that should be zero in qll queries
+	Z int64 `bson:"Z" zeek:"Z" zeektype:"count" json:"Z"`
+	// Answers contains the set of resource descriptions in the query answer
+	Answers []string `zeek:"answers" zeektype:"vector[string]" json:"answers"`
+	// TTLs contains a vector of interval type time to live values
+	TTLs []float64 `bson:"TTLs" zeek:"TTLs" zeektype:"vector[interval]" json:"TTLs"`
+	// Rejected indicates if this query was rejected or not
+	Rejected bool `zeek:"rejected" zeektype:"bool" json:"rejected"`
+	// AgentHostname names which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentHostname string `zeek:"agent_hostname" zeektype:"string" json:"agent_hostname"`
+	// AgentUUID identifies which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentUUID string `zeek:"agent_uuid" zeektype:"string" json:"agent_uuid"`
+	// Path of log file containing this record
+	LogPath string
+}
+
+func (d *DNS) SetLogPath(path string) { d.LogPath = path }

importer/zeektypes/http.go

@@ -0,0 +1,80 @@
+package zeektypes
+
+// EntryTypeHTTP should be matched against zeekFile.EntryType()
+// before using OpenZeekReader[ZeekHTTP](fs, zeekFile) to read from the file.
+const EntryTypeHTTP = "http"
+
+// HTTP provides a data structure for entries in zeek's HTTP log file
+type HTTP struct {
+	// TimeStamp of this connection
+	TimeStamp Timestamp `zeek:"ts" zeektype:"time" json:"ts"`
+	// UID is the Unique Id for this connection (generated by zeek)
+	UID string `zeek:"uid" zeektype:"string" json:"uid"`
+	// Source is the source address for this connection
+	Source string `zeek:"id.orig_h" zeektype:"addr" json:"id.orig_h"`
+	// SourcePort is the source port of this connection
+	SourcePort int `zeek:"id.orig_p" zeektype:"port" json:"id.orig_p"`
+	// Destination is the destination of the connection
+	Destination string `zeek:"id.resp_h" zeektype:"addr" json:"id.resp_h"`
+	// DestinationPort is the port at the destination host
+	DestinationPort int `zeek:"id.resp_p" zeektype:"port" json:"id.resp_p"`
+	// Transdepth is the ordinal value of requests into a pipeline transaction
+	TransDepth int64 `zeek:"trans_depth" zeektype:"count" json:"trans_depth"`
+	// Method is the request method used
+	Method string `zeek:"method" zeektype:"string" json:"method"`
+	// Host is the value of the HOST header
+	Host string `zeek:"host" zeektype:"string" json:"host"`
+	// URI is the uri used in this request
+	URI string `zeek:"uri" zeektype:"string" json:"uri"`
+	// Referrer is the value of the referrer header in the request
+	Referrer string `zeek:"referrer" zeektype:"string" json:"referrer"`
+	// Version is the value of version in the request
+	Version string `zeek:"version" zeektype:"string" json:"version"`
+	// UserAgent gives the user agent from the request
+	UserAgent string `zeek:"user_agent" zeektype:"string" json:"user_agent"`
+	// Origin gives the value of the origin header from the client
+	Origin string `zeek:"origin" zeektype:"string" json:"origin"`
+	// ReqLen holds the length of the request body uncompressed
+	ReqLen int64 `zeek:"request_body_len" zeektype:"count" json:"request_body_len"`
+	// RespLen hodls the length of the response body uncompressed
+	RespLen int64 `zeek:"response_body_len" zeektype:"count" json:"response_body_len"`
+	// StatusCode holds the status result
+	StatusCode int64 `zeek:"status_code" zeektype:"count" json:"status_code"`
+	// StatusMsg contains a string status message returned by the server
+	StatusMsg string `zeek:"status_msg" zeektype:"string" json:"status_msg"`
+	// InfoCode holds the last seen 1xx informational reply code
+	InfoCode int64 `zeek:"info_code" zeektype:"count" json:"info_code"`
+	// InfoMsg holds the last seen 1xx message string
+	InfoMsg string `zeek:"info_msg" zeektype:"string" json:"info_msg"`
+	// Tags contains a set of indicators of various attributes related to a particular req and
+	// response pair
+	// Tags []string `zeek:"tags" zeektype:"set[enum]" json:"tags"`
+	// UserName will contain a username in the case of basic auth implementation
+	UserName string `zeek:"username" zeektype:"string" json:"username"`
+	// Password will contain a password in the case of basic auth implementation
+	Password string `zeek:"password" zeektype:"string" json:"password"`
+	// Proxied contains all headers that indicate a request was proxied
+	Proxied []string `zeek:"proxied" zeektype:"set[string]" json:"proxied"`
+	// OrigFuids contains an ordered vector of uniq file IDs
+	OrigFuids []string `zeek:"orig_fuids" zeektype:"vector[string]" json:"orig_fuids"`
+	// OrigFilenames contains an ordered vector of filenames from the client
+	OrigFilenames []string `zeek:"orig_filenames" zeektype:"vector[string]" json:"orig_filenames"`
+	// OrigMimeTypes contains an ordered vector of mimetypes
+	OrigMimeTypes []string `zeek:"orig_mime_types" zeektype:"vector[string]" json:"orig_mime_types"`
+	// RespFuids contains an ordered vector of unique file IDs in the response
+	RespFuids []string `zeek:"resp_fuids" zeektype:"vector[string]" json:"resp_fuids"`
+	// RespFilenames contains an ordered vector of unique files in the response
+	RespFilenames []string `zeek:"resp_filenames" zeektype:"vector[string]" json:"resp_filenames"`
+	// RespMimeTypes contains an ordered vector of unique MIME entities in the HTTP response body
+	RespMimeTypes []string `zeek:"resp_mime_types" zeektype:"vector[string]" json:"resp_mime_types"`
+	// RespMimeTypes contains an ordered vector of unique MIME entities in the HTTP response body
+	ClientHeaderNames []string `zeek:"client_header_names" zeektype:"vector[string]" json:"client_header_names"`
+	// AgentHostname names which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentHostname string `zeek:"agent_hostname" zeektype:"string" json:"agent_hostname"`
+	// AgentUUID identifies which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentUUID string `zeek:"agent_uuid" zeektype:"string" json:"agent_uuid"`
+	// Path of log file containing this record
+	LogPath string
+}
+
+func (h *HTTP) SetLogPath(path string) { h.LogPath = path }

importer/zeektypes/ssl.go

@@ -0,0 +1,99 @@
+package zeektypes
+
+// EntryTypeSSL should be matched against zeekFile.EntryType()
+// before using OpenZeekReader[ZeekSSL](fs, zeekFile) to read from the file.
+const EntryTypeSSL = "ssl"
+
+// SSL provides a data structure for zeek's connection data
+type SSL struct {
+	// TimeStamp of this connection
+	TimeStamp Timestamp `zeek:"ts" zeektype:"time" json:"ts"`
+	// UID is the Unique Id for this connection (generated by zeek)
+	UID string `zeek:"uid" zeektype:"string" json:"uid"`
+	// Source is the source address for this connection
+	Source string `zeek:"id.orig_h" zeektype:"addr" json:"id.orig_h"`
+	// SourcePort is the source port of this connection
+	SourcePort int `zeek:"id.orig_p" zeektype:"port" json:"id.orig_p"`
+	// Destination is the destination of the connection
+	Destination string `zeek:"id.resp_h" zeektype:"addr" json:"id.resp_h"`
+	// DestinationPort is the port at the destination host
+	DestinationPort int `zeek:"id.resp_p" zeektype:"port" json:"id.resp_p"`
+	// VersionNum  : Numeric SSL/TLS version that the server chose
+	VersionNum int `zeek:"version_num" zeektype:"count" json:"version_num"`
+	// Version : SSL/TLS version that the server chose
+	Version string `zeek:"version" zeektype:"string" json:"version"`
+	// Cipher : SSL/TLS cipher suite that the server chose
+	Cipher string `zeek:"cipher" zeektype:"string" json:"cipher"`
+	// Curve : Elliptic curve the server chose when using ECDH/ECDHE
+	Curve string `zeek:"curve" zeektype:"string" json:"curve"`
+	// ServerName : Value of the Server Name Indicator SSL/TLS extension.
+	// It indicates the server name that the client was requesting.
+	ServerName string `zeek:"server_name" zeektype:"string" json:"server_name"`
+	// SessionID : Session ID offered by the client for session resumption.
+	// Not used for logging.
+	SessionID string `zeek:"session_id" zeektype:"string" json:"session_id"`
+	// Resumed : Flag to indicate if the session was resumed reusing the key
+	// material exchanged in an earlier connection
+	Resumed bool `zeek:"resumed" zeektype:"bool" json:"resumed"`
+	// ClientTicketEmptySessionSeen : Flag to indicate if we saw a non-empty
+	// session ticket being sent by the client using an empty session ID.
+	// This value is used to determine if a session is being resumed.
+	// It’s not logged.  Note: may not be present in older zeek versions.
+	ClientTicketEmptySessionSeen bool `zeek:"client_ticket_empty_session_seen" zeektype:"bool" json:"client_ticket_empty_session_seen"`
+	// ClientKeyExchangeSeen :Flag to indicate if we saw a client key exchange
+	// message sent by the client. This value is used to determine if a session
+	// is being resumed. It’s not logged.
+	// Note: may not be present in older zeek versions.
+	ClientKeyExchangeSeen bool `zeek:"client_key_exchange_seen" zeektype:"bool" json:"client_key_exchange_seen"`
+	// ServerAppData : Count to track if the server already sent an application
+	// data packet for TLS 1.3. Used to track when a session was established
+	// Note: may not be present in older zeek versions.
+	ServerAppData int `zeek:"server_appdata" zeektype:"count" json:"server_appdata"`
+	// ClientAppData : Flag to track if the client already sent an application
+	// data packet for TLS 1.3. Used to track when a session was established
+	// Note: may not be present in older zeek versions.
+	ClientAppData bool `zeek:"client_appdata" zeektype:"bool" json:"client_appdata"`
+	// LastAlert : Last alert that was seen during the connection.
+	LastAlert string `zeek:"last_alert" zeektype:"string" json:"last_alert"`
+	// NextProtocol : Next protocol the server chose using the application layer
+	// next protocol extension, if present.
+	NextProtocol string `zeek:"next_protocol" zeektype:"string" json:"next_protocol"`
+	// AnalyzerID : The analyzer ID used for the analyzer instance attached to
+	// each connection. It is not used for logging since it’s a meaningless
+	// arbitrary number. Note: may not be present in older zeek versions.
+	AnalyzerID int `zeek:"analyzer_id" zeektype:"count" json:"analyzer_id"`
+	// Established : Flag to indicate if this ssl session has been established
+	// successfully, or if it was aborted during the handshake
+	Established bool `zeek:"established" zeektype:"bool" json:"established"`
+	// Logged : Flag to indicate if this record already has been logged, to
+	// prevent duplicates. Note: may not be present in older zeek versions.
+	Logged bool `zeek:"logged" zeektype:"bool" json:"logged"`
+	// CertChainFuids
+	CertChainFuids []string `zeek:"cert_chain_fuids" zeektype:"vector[string]" json:"cert_chain_fuids"`
+	// ClientCertChainFuids
+	ClientCertChainFuids []string `zeek:"client_cert_chain_fuids" zeektype:"vector[string]" json:"client_cert_chain_fuids"`
+	// Subject
+	Subject string `zeek:"subject" zeektype:"string" json:"subject"`
+	// Issuer
+	Issuer string `zeek:"issuer" zeektype:"string" json:"issuer"`
+	// ClientSubject
+	ClientSubject string `zeek:"client_subject" zeektype:"string" json:"client_subject"`
+	// ClientIssuer
+	ClientIssuer string `zeek:"client_issuer" zeektype:"string" json:"client_issuer"`
+	// ValidationStatus
+	ValidationStatus string `zeek:"validation_status" zeektype:"string" json:"validation_status"`
+	// ValidationCode  : Numeric SSL/TLS version that the server chose
+	ValidationCode int `zeek:"validation_code" zeektype:"int" json:"validation_code"`
+	// JA3 client hash
+	JA3 string `bson:"ja3" zeek:"ja3" zeektype:"string" json:"ja3"`
+	// JA3S server hash
+	JA3S string `bson:"ja3s" zeek:"ja3s" zeektype:"string" json:"ja3s"`
+	// AgentHostname names which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentHostname string `zeek:"agent_hostname" zeektype:"string" json:"agent_hostname"`
+	// AgentUUID identifies which sensor recorded this event. Only set when combining logs from multiple sensors.
+	AgentUUID string `zeek:"agent_uuid" zeektype:"string" json:"agent_uuid"`
+	// Path of log file containing this record
+	LogPath string
+}
+
+func (s *SSL) SetLogPath(path string) { s.LogPath = path }

installer/Installer.md

@@ -0,0 +1,32 @@
+```
+rita-<version>.tar.gz
+│   install_rita.yml
+│   install_rita.sh
+│
+└───/scripts
+│    │   ansible-installer.sh
+│    │   helper.sh
+│    │   sshprep
+│    
+└───/files
+│   │   
+│   │   rita-<version>-image.tar
+│   │   docker-compose
+│   │
+│   └───/opt
+│   │   │   docker-compose.yml
+│   │   │   .env
+│   │   │   README
+│   │   │   LICENSE
+│   │   │   rita.sh
+│   │
+│   └───/etc
+│       │   config.hjson
+│       │   config.xml
+│       │   http_extensions_list.csv
+│       │   logger-cron
+│       │   syslog-ng.conf
+│       │   timezone.xml
+│       └───/threat_intel_feeds
+
+```

installer/build_image.sh

@@ -0,0 +1,5 @@
+# local test script to build RITA as an amd64 Docker image and export it to a file
+VERSION=$(git describe --always --abbrev=0 --tags)
+
+sudo docker buildx build  --platform linux/amd64 --tag ghcr.io/activecm/rita-v2:"$VERSION" ../
+docker save -o rita-v2-"$VERSION"-image.tar ghcr.io/activecm/rita-v2:"$VERSION"

installer/generate_installer.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+set -e
+
+# Generates the RITA installer by creating a temporary folder in the current directory named 'stage'
+# and copies files that must be in the installer into the stage folder.
+# Once all directories are placed in stage, it is compressed and stage is deleted
+
+# get RITA version from git
+VERSION=$(git describe --always --abbrev=0 --tags)
+echo "Generating installer for RITA $VERSION..."
+
+
+# change working directory to directory of this script
+pushd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" > /dev/null
+
+BASE_DIR="./rita-$VERSION-installer" # was ./stage/bin
+
+# create staging folder
+rm -rf "$BASE_DIR"
+# mkdir ./stage
+
+# create ansible subfolders
+# ANSIBLE_FILES=./stage/.ansible/files
+SCRIPTS="$BASE_DIR/scripts"
+ANSIBLE_FILES="$BASE_DIR/files"
+ANSIBLE_PLAYBOOKS="$BASE_DIR/.ansible/playbooks"
+
+mkdir "$BASE_DIR"
+mkdir -p "$ANSIBLE_FILES"
+mkdir -p "$SCRIPTS"
+mkdir -p "$ANSIBLE_PLAYBOOKS"
+
+# create subfolders (for files that installed RITA will contain)
+INSTALL_OPT="$ANSIBLE_FILES"/opt
+INSTALL_ETC="$ANSIBLE_FILES"/etc
+mkdir "$ANSIBLE_FILES"/opt
+mkdir "$ANSIBLE_FILES"/etc
+
+
+# copy files in base dir
+cp ./install_scripts/install_rita.yml "$BASE_DIR"
+cp ./install_scripts/install_rita.sh "$BASE_DIR" # entrypoint
+
+# copy files to helper script folder
+cp ./install_scripts/ansible-installer.sh "$SCRIPTS"
+cp ./install_scripts/helper.sh "$SCRIPTS"
+cp ./install_scripts/sshprep "$SCRIPTS"
+
+# copy files to the ansible files folder
+cp ./install_scripts/docker-compose "$ANSIBLE_FILES" # docker-compose v1 backwards compatibility script
+
+
+# copy over configuration files to /files/etc
+cp -r ../deployment/* "$INSTALL_ETC"
+cp ../default_config.hjson "$INSTALL_ETC"/config.hjson
+
+# copy over installed files to /opt
+cp ../rita.sh "$INSTALL_OPT"/rita.sh
+cp ../.env.production "$INSTALL_OPT"/.env
+cp ../docker-compose.prod.yml "$INSTALL_OPT"/docker-compose.yml
+cp ../LICENSE "$INSTALL_OPT"/LICENSE
+cp ../README.md "$INSTALL_OPT"/README
+
+
+
+# update version variables for files that need them
+if [ "$(uname)" == "Darwin" ]; then
+    sed -i'.bak' "s/REPLACE_ME/${VERSION}/g" "$BASE_DIR/install_rita.yml" # WAS $ANSIBLE_PLAYBOOKS
+    sed -i'.bak' "s/REPLACE_ME/${VERSION}/g" "$BASE_DIR/install_rita.sh"
+    sed -i'.bak' "s#ghcr.io/activecm/rita-v2:latest#ghcr.io/activecm/rita-v2:${VERSION}#g" "$INSTALL_OPT/docker-compose.yml"
+
+    rm "$BASE_DIR/install_rita.yml.bak"
+    rm "$BASE_DIR/install_rita.sh.bak"
+    rm "$INSTALL_OPT/docker-compose.yml.bak"
+else 
+    sed -i  "s/REPLACE_ME/${VERSION}/g" "$BASE_DIR/install_rita.yml" # WAS $ANSIBLE_PLAYBOOKS
+    sed -i  "s/REPLACE_ME/${VERSION}/g" "$BASE_DIR/install_rita.sh"
+    sed -i  "s#ghcr.io/activecm/rita-v2:latest#ghcr.io/activecm/rita-v2:${VERSION}#g" "$INSTALL_OPT/docker-compose.yml"
+fi
+
+
+
+
+# TODO remove when repo is public
+./build_image.sh
+cp "./rita-v2-$VERSION-image.tar" "$ANSIBLE_FILES" # was $INSTALL_OPT
+rm "./rita-v2-$VERSION-image.tar"
+
+# create tar
+# TODO the inner folder is named stage, should be rita-$VERSION
+tar -czf "rita-$VERSION.tar.gz" "$BASE_DIR"
+
+# delete staging folder
+rm -rf "$BASE_DIR"
+
+# switch back to original working directory
+popd > /dev/null
+
+echo "Finished generating installer."

installer/install_scripts/ansible-installer.sh

@@ -0,0 +1,347 @@
+#!/bin/bash
+#Copyright 2024, Active Countermeasures
+#Written by WS with guidance from NG
+pushd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" > /dev/null
+
+
+source ./helper.sh
+
+#This script installs ansible and supporting tools needed for rita
+#and/or AC-Hunter on a deb, rpm, port, or brew package -based system. 
+#It also patches all installed packages.
+
+#The general aim is that this will work on multiple Linux distributions
+#that use either .deb or .rpm packages, though more testing is needed.
+#Please contact bill@activecountermeasures.com if you have any updates
+#on errors or compatibility issues found.  Many thanks to NG for 
+#the original idea and multiple improvements.
+
+
+#Tested on:
+#FIXME
+
+ansible_installer_version="0.3.6"
+
+#Uncomment one of the following lines to set the default program to download and install
+data_needed="rita"
+#data_needed="achunter"
+
+
+
+
+require_sudo() {
+	#Stops the script if the user does not have root priviledges and cannot sudo
+	#Additionally, sets $SUDO to "sudo" and $SUDO_E to "sudo -E" if needed.
+
+	status "Checking sudo"		#================
+	if [ "$EUID" -eq 0 ]; then
+		SUDO=""
+		SUDO_E=""
+		return 0
+	elif sudo -v; then			#Confirms I'm allowed to run commands via sudo
+		SUDO="sudo"
+		SUDO_E="sudo -E"
+		return 0
+	else
+		#I'm _not_ allowed to run commands as sudo.
+		echo "It does not appear that user $USER has permission to run commands under sudo." >&2
+		if grep -q '^wheel:' /etc/group ; then
+			fail "Please run    usermod -aG wheel $USER   as root, log out, log back in, and retry the install"
+		elif grep -q '^sudo:' /etc/group ; then
+			fail "Please run    usermod -aG sudo $USER   as root, log out, log back in, and retry the install"
+		else
+			fail "Please give this user the ability to run commands as root under sudo, log out, log back in, and retry the install"
+		fi
+	fi
+}
+
+
+tmp_dir() {
+	mkdir -p "$HOME/tmp/"
+	tdirname=`mktemp -d -q "$HOME/tmp/install-tools.XXXXXXXX" </dev/null`
+	if [ ! -d "$tdirname" ]; then
+		fail "Unable to create temporary directory."
+	fi
+	echo "$tdirname"
+}
+
+enable_repositories() {
+	status "Enable additional repository/repositories"		#================
+
+	if [ ! -s /etc/os-release ]; then
+		fail "Unable to read /etc/os-release"
+	else
+		. /etc/os-release
+		case "$ID/$VERSION_ID" in
+		alma/8|rocky/8)
+			dnf config-manager --set-enabled powertools
+			dnf install epel-release
+			;;
+		alma/9|rocky/9)
+			dnf config-manager --set-enabled crb
+			dnf install epel-release
+			;;
+		centos/7)
+			yum install epel-release
+			;;
+		centos/8)
+			dnf config-manager --set-enabled powertools
+			dnf install epel-release epel-next-release
+			;;
+		centos/9)
+			dnf config-manager --set-enabled crb
+			dnf install epel-release epel-next-release
+			;;
+		rhel/7)
+			subscription-manager repos --enable rhel-*-optional-rpms --enable rhel-*-extras-rpms --enable rhel-ha-for-rhel-*-server-rpms
+			yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+			;;
+		rhel/8)
+			subscription-manager repos --enable codeready-builder-for-rhel-8-$(arch)-rpms
+			dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+			;;
+		rhel/9)
+			subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms
+			dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+			;;
+		fedora/*)
+			:										#It does not appear that fedora needs any extra repositories
+			;;
+		ubuntu/*)
+			sudo apt update
+			sudo apt install software-properties-common || sudo apt install python-software-properties
+			sudo add-apt-repository --yes --update ppa:ansible/ansible
+			;;
+		*)
+			fail "unknown OS $ID/$VERSION_ID"
+			;;
+		esac
+	fi
+}
+
+patch_system() {
+	#Make sure all currently installed packages are updated.  This has the added benefit
+	#that we update the package metadata for later installing new packages.
+
+	status "Patching system"		#================
+	if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then
+		while ! $SUDO sudo add-apt-repository universe ; do
+			echo "Error subscribing to universe repository, perhaps because a system update is running; will wait 60 seconds and try again." >&2
+			sleep 60
+		done
+		while ! $SUDO apt-get -q -y update >/dev/null ; do
+			echo "Error updating package metadata, perhaps because a system update is running; will wait 60 seconds and try again." >&2
+			sleep 60
+		done
+		while ! $SUDO apt-get -q -y upgrade >/dev/null ; do
+			echo "Error updating packages, perhaps because a system update is running; will wait 60 seconds and try again." >&2
+			sleep 60
+		done
+		while ! $SUDO apt-get -q -y install lsb-release >/dev/null ; do
+			echo "Error installing lsb-release, perhaps because a system update is running; will wait 60 seconds and try again." >&2
+			sleep 60
+		done
+	elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then
+		$SUDO yum -q -e 0 makecache
+		$SUDO yum -y -q -e 0 -y install deltarpm
+		$SUDO yum -q -e 0 -y update
+		$SUDO yum -y -q -e 0 -y install redhat-lsb-core yum-utils
+		if [ -s /etc/redhat-release -a -s /etc/os-release ]; then
+			. /etc/os-release
+			if [ "$VERSION_ID" = "7" ]; then
+				$SUDO yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+				if [ ! -e /etc/centos-release ]; then
+					$SUDO yum -y install subscription-manager
+					$SUDO subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"  --enable "rhel-ha-for-rhel-*-server-rpms"
+				fi
+			elif [ "$VERSION_ID" = "8" ]; then
+				$SUDO yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+				if [ -e /etc/centos-release ]; then
+					$SUDO dnf config-manager --set-enabled powertools
+				else
+					$SUDO yum -y install subscription-manager
+					$SUDO subscription-manager repos --enable "codeready-builder-for-rhel-8-`/bin/arch`-rpms"
+				fi
+			fi
+		fi
+		$SUDO yum -q -e 0 makecache
+	fi
+}
+
+
+install_tool() {
+	#Install a program.  $1 holds the name of the executable we need
+	#$2 is one or more packages that can supply that executable (put preferred package names early in the list).
+
+
+	binary="$1"
+	potential_packages="$2"
+
+	if type -path "$binary" >/dev/null ; then
+		status "== $binary executable is installed."		#================
+	else
+		status "== Installing package that contains $binary"		#================
+		if [ -x /usr/bin/apt-get -a -x /usr/bin/dpkg-query ]; then
+			for one_package in $potential_packages ; do
+				if ! type -path "$binary" >/dev/null ; then		#if a previous package was successfully able to install, don't try again.
+					$SUDO apt-get -q -y install $one_package
+				fi
+			done
+		elif [ -x /usr/bin/yum -a -x /bin/rpm ]; then
+			#Yum takes care of the lock loop for us
+			for one_package in $potential_packages ; do
+				if ! type -path "$binary" >/dev/null ; then		#if a previous package was successfully able to install, don't try again.
+					$SUDO yum -y -q -e 0 install $one_package
+				fi
+			done
+		else
+			fail "Neither (apt-get and dpkg-query) nor (yum, rpm, and yum-config-manager) is installed on the system"
+		fi
+	fi
+
+	if type -path "$binary" >/dev/null ; then
+		return 0
+	else
+		echo "WARNING: Unable to install $binary from a system package" >&2
+		return 1
+	fi
+}
+
+echo "ansible_installer version $ansible_installer_version" >&2
+
+if [ -n "$1" ]; then
+	if [ "$1" = "rita" ]; then
+		data_needed="rita"
+	elif [ "$1" = "achunter" ]; then
+		data_needed="achunter"
+	else
+		echo "I do not recognize the command line parameter you specified - please put   rita   or   achunter   as the first command line parameter to say which program you need installed, followed by the host on which you want rita installed.  Exiting."
+		exit 1
+	fi
+fi
+if [ -n "$2" ]; then
+	install_target="$2"
+else
+	install_target="localhost"
+fi
+
+require_sudo
+
+# check if macOS
+if [ "$(uname)" == "Darwin" ]; then
+	# check if ansible is installed
+	which -s ansible
+	if [[ $? != 0 ]] ; then
+		# check if homebrew is installed
+		which -s brew
+		if [[ $? != 0 ]] ; then
+			fail "Homebrew is required to install Ansible."
+		fi
+		# install ansible via homebrew
+		echo "Installing Ansible via brew..."
+		brew install ansible
+	else 
+		echo "== Ansible is already installed."
+	fi
+	# FIXME
+	# exit to avoid fubaring mac
+	# fail "bingbong"
+else
+	patch_system
+
+	enable_repositories
+
+
+	status "Installing needed tools"		#================
+	install_tool python3 "python3"
+	install_tool pip3 "python3-pip"
+	python3 -m pip -V ; retcode="$?"
+	if [ "$retcode" != 0 ]; then
+		fail "Unable to run python3's pip, exiting."
+	fi
+
+
+	install_tool wget "wget"
+	install_tool curl "curl"
+	install_tool sha256sum "coreutils"
+	install_tool ansible "ansible ansible-core"
+fi
+
+
+#We need to install zeek through the rita installer in order to install both
+#install_tool zeek "zeek"
+#install_tool zeekctl "zeekctl"
+
+
+
+
+
+status "Preparing this system"		#================
+#Try to add /usr/local/bin/ to path (though the better way is to log out and log back in)
+if ! echo "$PATH" | grep -q '/usr/local/bin' ; then
+	echo "Adding /usr/local/bin to path" >&2
+	#For this login only...
+	export PATH="$PATH:/usr/local/bin/"
+	#...and for future logins
+	if [ -s /etc/environment ]; then
+		echo 'export PATH="$PATH:/usr/local/bin/"' | sudo tee -a /etc/environment >/dev/null
+	elif [ -s /etc/profile ]; then
+		echo 'export PATH="$PATH:/usr/local/bin/"' | sudo tee -a /etc/profile >/dev/null
+	else
+		echo "Unable to add /usr/local/bin/ to path." >&2
+	fi
+fi
+
+#ansible-galaxy install community.docker		#FIXME Removeme
+ansible-galaxy collection install community.docker --force
+
+
+# if [ "$data_needed" = "rita" ]; then
+	#This may not be needed with ansible-playbook's "-i" param followed by a comma separated list of hosts that ends in a comma
+	##FIXME - this won't support a comma separated list of hosts, forcing us to install to one remote target at a time.
+	#if [ -d "/opt/local/etc/ansible/" ]; then
+	#	ans_hosts="/opt/local/etc/ansible/hosts"
+	#elif [ -d "/etc/ansible" ]; then
+	#	ans_hosts="/etc/ansible/hosts"
+	#else
+	#	echo "Unable to locate ansible configuration directory to manage the hosts file, exiting."
+	#	exit 1
+	#fi
+	#if ! grep -q '^'"$install_target"'$' "$ans_hosts"
+	#	#There's no entry for this host in the ansible hosts file, we must add it.
+	#	echo "" >>"$ans_hosts"
+	#	echo "#Added by the rita installer" >>"$ans_hosts"
+	#	echo '['"${install_target}-group"']' >>"$ans_hosts"
+	#	echo "$install_target" >>"$ans_hosts"
+	#	echo "" >>"$ans_hosts"
+	#fi
+
+	# status "Installing rita via ansible on $install_target"		#================
+	# if [ "$install_target" = "localhost" -o "$install_target" = "127.0.0.1" -o "$install_target" = "::1" ]; then
+	# 	ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," ~/.ansible/playbooks/install_rita.yml
+	# else
+	# 	status "Setting up future ssh connections to $install_target .  You may be asked to provide your ssh password to this system."		#================
+	# 	sshprep "$install_target"
+	# 	ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," ~/.ansible/playbooks/install_rita.yml
+	# fi
+
+# elif [ "$data_needed" = "achunter" ]; then
+# 	echo 'Not implemented yet, exiting.' >&2
+# else
+# 	echo 'I do not know what program to install, skipping.' >&2
+# fi
+
+# echo "Unless you see warnings above that an install failed, you should have RITA installed." >&2
+# echo '!!!!!!!!!!!!You must log out and log back in to make sure your PATH is set correctly!!!!!!!!!!!!' >&2
+
+touch "$HOME/rita-installed.flag"
+
+
+
+
+
+
+
+
+
+popd > /dev/null

installer/install_scripts/docker-compose

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker compose "$@"

installer/install_scripts/helper.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+
+RED=$(tput setaf 1)
+YELLOW=$(tput setaf 3)
+NORMAL=$(tput sgr0)
+
+
+fail() {
+	#Something failed, exit.
+
+	echo "${RED}$@, exiting.${NORMAL}" >&2
+	exit 1
+}
+
+
+status() {
+	if [ "$verbose" = 'yes' ]; then
+		echo "== $@" >&2
+	fi
+}
+
+verbose="yes"

installer/install_scripts/install_rita.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+RITA_VERSION="REPLACE_ME"
+
+set -e 
+
+install_target="$1"
+
+# change working directory to directory of this script
+pushd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" > /dev/null
+
+source ./scripts/helper.sh
+
+
+./scripts/ansible-installer.sh
+
+
+status "Installing rita via ansible on $install_target"		#================
+if [ "$install_target" = "localhost" -o "$install_target" = "127.0.0.1" -o "$install_target" = "::1" ]; then
+        if [ "$(uname)" == "Darwin" ]; then
+            # TODO support macOS install target
+            echo "${YELLOW}Installing RITA via Ansible on the local system is not yet supported on MacOS.${NORMAL}"
+            exit 1
+        fi
+    ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," install_rita.yml
+else
+    status "Setting up future ssh connections to $install_target .  You may be asked to provide your ssh password to this system."		#================
+    ./scripts/sshprep "$install_target"
+    ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," install_rita.yml
+fi
+
+    # ansible-playbook -i ../digitalocean_inventory.py -e "install_hosts=all" install_rita.yml
+
+echo \
+"
+░█▀▀█ ▀█▀ ▀▀█▀▀ ─█▀▀█
+░█▄▄▀ ░█─ ─░█── ░█▄▄█
+░█─░█ ▄█▄ ─░█── ░█─░█ ${RITA_VERSION}
+
+Brought to you by Active CounterMeasures©
+"
+echo "RITA was successfully installed!"
+
+# switch back to original working directory
+popd > /dev/null

installer/install_scripts/install_rita.yml

@@ -0,0 +1,609 @@
+---
+#ansible install playbook for rita V2.
+#Version: 202406261404
+#sample runs:
+#	Optional: Add the following block, without #'s  to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports).
+#The hosts must each be on their own line.  These can be full or short hostnames or a name following "Host" in ~/.ssh/config .
+#
+#[allritas]
+#ro810
+#ub2404
+#
+#	Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list:
+#
+#	ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '	#-C (no changes) means do a dry run
+#	ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '
+
+#Many thanks to but-i-am-dominator for his help with this playbook.
+
+
+- name: "RITA Install: RITA installer and system prep and checks."
+  hosts: "{{  install_hosts  }}"
+  #hosts: "{{  install_hosts | default('all')  }}"						#Not a good idea to fall back on every host in your ansible hosts file.
+  become: true
+
+  vars:
+    rita_version: "REPLACE_ME"									#FIXME - replace with the right version
+    #rita_container_image: hello-world								# Test container to confirm that we can install images.
+    #rita_container_image: activecm/rita							# Confirm exact image name to pull
+    #rita_container_image: activecm/rita:4.3.1
+    rita_container_image: "ghcr.io/activecm/rita-v2:{{  rita_version  }}"
+    clickhouse_container_image: clickhouse/clickhouse-server:latest
+    ansible_python_interpreter: /bin/python3							# Centos 7 defaults to using python2, so we force python 3.  This change does not break any other distros
+
+#Early tasks needed to support the rest of the install
+  pre_tasks:
+#Known distribution?
+    - name: "RITA Install: Checking Linux distribution."
+      ansible.builtin.fail:
+        msg: "Distribution name: {{  ansible_distribution  }}  does not appear to be recognized - please contact ACM"
+      when: ( ansible_distribution != 'AlmaLinux' and ansible_distribution != 'CentOS' and ansible_distribution != 'Fedora' and ansible_distribution != 'OracleLinux' and ansible_distribution != 'Rocky' and ansible_distribution != 'Debian' and ansible_distribution != 'Ubuntu' and ansible_distribution != 'Kali' )
+      # and ansible_distribution != 'RedHat'
+      tags:
+        - linux
+
+    - name: "RITA Install: Checking Linux distribution version."
+      ansible.builtin.fail:
+        msg: "Warning: Linux distribution {{  ansible_distribution  }} {{  ansible_distribution_major_version  }} may not have been tested - please contact ACM and report whether the install worked or not"
+      when: ( ( ansible_distribution == 'AlmaLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'CentOS' and (ansible_distribution_major_version != '7' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Fedora' and (ansible_distribution_major_version != '40') ) or ( ansible_distribution == 'OracleLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Rocky' and (ansible_distribution_major_version != '8') ) or ( ansible_distribution == 'Debian' and (ansible_distribution_major_version != '12') ) or ( ansible_distribution == 'Kali' and (ansible_distribution_major_version != '2024') )  or ( ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version != '20' and ansible_distribution_major_version != '24') ) )
+      #or ( ansible_distribution != 'RedHat' and (ansible_distribution_major_version == '9') ) 
+      ignore_errors: True		#We print a warning but do not abort if this is an unknown combination of distribution and major version.
+      tags:
+        - linux
+
+#CPU Architecture
+    - name: "RITA Install: Check system architecture."
+      ansible.builtin.fail:
+        msg: "Unsupported CPU architecture: {{  ansible_architecture  }}"
+      when: ( ansible_architecture != "x86_64" ) 	#and ansible_architecture != "aarch64" ) # "aarch64" for pi.  #pi0w is armv6l.  i386.  amd64?
+
+#Selinux checks
+    - name: "RITA Install: /sys/fs/selinux/enforce  exists."
+      stat:
+        path: "/sys/fs/selinux/enforce"
+      check_mode: true
+      changed_when: false
+      register: selinuxenforce_check
+      tags:
+        - linux
+
+    - name: "RITA Install: sys filesystem check for selinux."
+      lineinfile:
+        path: /sys/fs/selinux/enforce
+        regexp: '^1'
+        line: 0
+        create: false
+        unsafe_writes: true									#Needed because the original file in the sys filesystem and Ansible's tmp directory are on different filesystems.
+        state: present
+      #check_mode: yes
+      changed_when: false
+      #register: enforce_check
+      when: selinuxenforce_check.stat.exists
+      tags:
+        - linux
+
+#Add tools needed by later stages
+    # Provides "needs-restarting" for ansible's ability to manage rebooting after patching
+    - name: "RITA Install: Check for yum-utils before proceeding."
+      command: rpm -qa | grep yum-utils
+      check_mode: true
+      changed_when: false
+      register: package_check
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    - name: "RITA Install: Install yum-utils if not found."
+      package:
+        name: yum-utils
+        state: latest
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and '"yum-utils" not in package_check'
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    # Install aptitude, preferred by ansible for package management on Debian/Ubuntu
+    - name: "RITA Install: Install aptitude on debian-based system."
+      apt:
+        name: aptitude
+        state: latest
+        update_cache: true
+        cache_valid_time: 3600
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' )		#While Kali is based on Debian, it does not include the aptitude package.
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+
+  tasks:
+# Make sure all rpm packages up to date, add packages
+    - name: "RITA Install: Patch and install packages on rpm-based servers."
+      block:
+        - name: "RITA Install: Patch all rpm-based servers."
+          yum:											#We use the "yum" module insteead of dnf to support rpm distros that only have yum
+            name: "*"
+            state: latest
+            skip_broken: yes
+            update_cache: yes
+          tags:
+            - packages
+            - linux
+            - linuxrpm
+
+        - name: "RITA Install: Install rpm packages on rpm-based distributions."
+          yum:
+            name:
+              - nano
+              - nmap-ncat
+              - dnf-plugins-core								#Provides config-manager binary on Fedora
+              - wget
+              - lshw										#For user troubleshooting
+              - net-tools									#For user troubleshooting
+            state: latest
+            update_cache: true
+          tags:
+            - packages
+            - linux
+            - linuxrpm
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+
+    - name: "RITA Install: Install pip on Centos/Fedora."
+      yum:
+        name:
+          - python3-pip
+        state: latest
+        update_cache: true
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      #  or ansible_distribution == 'OracleLinux'		#Note: OracleLinux, and therefore SecurityOnion too, do not include pip3.  Disabled.
+
+    - name: "RITA Install: Patch and install packages on debian-based servers."
+      block:
+        - name: "RITA Install: Patch all debian-based servers."
+          apt:
+            name: "*"
+            state: latest
+            update_cache: yes
+            cache_valid_time: 3600
+          tags:
+            - packages
+            - linux
+            - linuxdeb
+
+        - name: "RITA Install: Install apt packages on deb-based distributions."
+          apt:
+            pkg:
+              - nano
+              #Following are to support docker
+              - apt-transport-https
+              - ca-certificates
+              - curl
+              - python3-pip
+              - python3-setuptools
+              - wget
+              #Following is for user troubleshooting
+              - net-tools
+            state: latest
+            update_cache: true
+            cache_valid_time: 3600
+          tags:
+            - packages
+            - linux
+            - linuxdeb
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Ubuntu' )
+
+
+    - name: "RITA Install: Install packages on Debian and Ubuntu."
+      apt:
+        pkg:
+          - ncat				#"ncat" is nmap's netcat on Ubuntu and Debian, listd but not available on Kali
+          - software-properties-common
+          - virtualenv
+          - lshw				#listed, but somehow not available on Kali
+        state: latest
+        update_cache: true
+        cache_valid_time: 3600
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' )
+
+    - name: "RITA Install: Install packages on Kali."
+      apt:
+        pkg:
+          - netcat-traditional
+          - python3-virtualenv
+        state: latest
+        update_cache: true
+        cache_valid_time: 3600
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+      when: ( ansible_distribution == 'Kali' )
+
+
+
+#Add repositories
+    - name: "RITA Install: Add Docker Ubuntu GPG apt key."
+      apt_key:
+        url: https://download.docker.com/linux/ubuntu/gpg
+        state: present
+      when: ( ansible_distribution == 'Ubuntu' )
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Install: Add Docker Debian GPG apt key."
+      apt_key:
+        url: https://download.docker.com/linux/debian/gpg
+        state: present
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' )
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Install: Add Docker Repository to Ubuntu or Debian."
+      apt_repository:
+        repo: deb https://download.docker.com/linux/{{  ansible_distribution|lower  }} {{  ansible_distribution_release  }} stable
+        state: present
+      when: ( ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' )
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Install: Add Docker Repository to Kali."
+      apt_repository:
+        repo: deb https://download.docker.com/linux/debian bookworm stable
+        state: present
+      when: ( ansible_distribution == 'Kali' and ansible_distribution_major_version == '2024' )
+      #Debian bookworm appears to be the right one to use according to https://www.kali.org/docs/containers/installing-docker-on-kali/
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Install: Add Docker repository to Fedora distributions."
+      yum_repository:
+        name: docker-ce
+        description: Docker package repository
+        gpgkey: https://download.docker.com/linux/fedora/gpg
+        baseurl: https://download.docker.com/linux/fedora/$releasever/$basearch/stable/
+        state: present
+        enabled: true
+      when: ( ansible_distribution == 'Fedora' )	# and ansible_distribution_major_version == '40' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    - name: "RITA Install: Add Docker Repository to AlmaLinux/Centos/OracleLinux/Rocky distributions."
+      #shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+      yum_repository:
+        name: docker-ce
+        description: Docker package repository
+        gpgkey: https://download.docker.com/linux/centos/gpg
+        baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable/
+        state: present
+        enabled: true
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'Rocky' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    - name: "RITA Install: Add Docker Repository to RHEL distribution."
+      yum_repository:
+        name: docker-ce
+        description: Docker package repository
+        gpgkey: https://download.docker.com/linux/rhel/gpg
+        baseurl: https://download.docker.com/linux/rhel/$releasever/$basearch/stable/
+        state: present
+        enabled: true
+      when: ( ansible_distribution == 'RedHat' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+
+#Install docker
+    - name: "RITA Install: Install docker on debian-based distributions."
+      block:
+        - name: "RITA Install: Uninstall unofficial docker packages on debian-based distributions."
+          apt:
+            name:
+              - docker-client
+              - docker-client-latest
+              - docker-common
+              - docker-compose
+              - docker-compose-v2
+              - docker-doc
+              - docker-engine
+              - docker-latest
+              - docker-latest-logrotate
+              - docker-logrotate
+              - docker.io
+              - podman-docker
+            state: absent
+            update_cache: true
+            cache_valid_time: 3600
+          tags:
+            - docker
+            - linux
+            - linuxdeb
+
+        - name: "RITA Install: Install docker-ce on debian-based distributions."
+          apt:
+            name:
+              - docker-ce
+              - docker-ce-cli
+              - docker-compose-plugin
+              - containerd.io
+            state: latest
+            update_cache: true
+            cache_valid_time: 3600
+          tags:
+            - docker
+            - linux
+            - linuxdeb
+
+        - name: "RITA Install: Install docker modules for Python on deb-based distributions."
+          apt:
+            name:
+              - python3-docker
+              - python3-requests								#We'll have to see if debian/ubuntu can work with the stock (2.28.1 in debian 12.05 / 2.31.0 in ubuntu 24.04)
+          tags:
+            - docker
+            - linux
+            - linuxdeb
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Ubuntu')
+
+
+    - name: "RITA Install: Install docker on rpm-based distributions."
+      block:
+        - name: "RITA Install: Uninstall unofficial docker packages on rpm-based distributions."
+          yum:
+            name:
+              - docker-client
+              - docker-client-latest
+              - docker-common
+              - docker-compose
+              - docker-compose-v2
+              - docker-doc
+              - docker-engine-selinux
+              - docker-engine
+              - docker-latest
+              - docker-latest-logrotate
+              - docker-logrotate
+              - docker-selinux
+              - docker.io
+              - docker
+              - podman-docker
+              - podman
+              - runc
+            state: absent
+            update_cache: true
+          tags:
+            - docker
+            - linux
+            - linuxrpm
+
+        - name: "RITA Install: Install docker-ce on rpm-based distributions."
+          yum:
+            name:
+              - docker-ce
+              - docker-ce-cli
+              - docker-buildx-plugin
+              - docker-compose-plugin
+              - containerd.io
+            state: latest
+            update_cache: true
+          tags:
+            - docker
+            - linux
+            - linuxrpm
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      #Reminder that RedHat/RHEL 9 is broken as of 20240618
+
+
+    - name: "RITA Install: replace python3-requests with a new version installed by pip."
+      block:
+        - name: "RITA Install: Uninstall unofficial docker packages on rpm-based distributions."
+          yum:
+            name:
+              - python3-requests								#As of 20240618, issue with requests code: "Error connecting: Error while fetching server API version: Not supported URL scheme http+docker".  Installing requests with pip appears to install a newer version that handles the issue.
+            state: absent
+            update_cache: true
+          tags:
+            - docker
+            - linux
+            - linuxrpm
+
+        - name: "RITA Install: Install docker modules for Python on rpm-based distributions."
+          pip:
+            name:
+              - docker
+              - requests
+          tags:
+            - docker
+            - linux
+            - linuxrpm
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      #OracleLinux and SecurityOnion don't include pip so we can't do these steps there.
+
+
+    - name: "RITA Install: Start and enable docker in systemd."
+      systemd:
+        name: docker
+        state: started
+        enabled: yes
+      when: ( ansible_distribution != 'OracleLinux' )
+      tags:
+        - docker
+        - linux
+        - linuxdeb
+        - linuxrpm
+      #It appears the "docker modules for python on rpm-based linux" is needed to use the ansible "systemd" module, so we can't use that module on OracleLinux...
+
+      #...so we fall back on starting and enabling it on OracleLinux by hand.
+    - name: "RITA Install: Start and enable docker in systemd on OracleLinux."
+      shell: systemctl enable docker.service ; systemctl start docker.service
+      when: ( ansible_distribution == 'OracleLinux' )
+      tags:
+        - docker
+        - linux
+        - linuxrpm
+
+    - name: "RITA Install: Transfer docker-compose script to target system for backwards compatibility."
+      copy:
+        src: docker-compose
+        dest: /usr/local/bin/docker-compose
+        owner: root
+        group: root
+        mode: 0755
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+#Make directories
+    - name: "RITA Install: Create configuration directories."
+      ansible.builtin.file:
+        path: "{{  item  }}"
+        state: directory
+        owner: root		#FIXME - check
+        group: root		#FIXME - check
+        mode: 0755
+      loop:
+        - /etc/rita/
+        - /opt/rita/
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+#Install RITA
+    #Following pulls right from dockerhub.  We may not be able to do this if the system is airgapped
+    - name: "RITA Install: Install {{  rita_container_image  }} docker image."
+      block:
+        - name: "Pull from Github.io Container repo"
+          community.docker.docker_image:
+            name: "{{  rita_container_image  }}"
+            source: pull
+            force_source: true
+      rescue:
+        - name: "RITA Install: Transfer RITA container image to target system"
+          copy:
+          # TODO remove -v2
+            src: "rita-v2-{{  rita_version  }}-image.tar"
+            dest: /opt/rita
+            owner: root
+            group: root
+            mode: 0644
+        - name: "Install {{ rita_container_name }} container image from file"
+          community.docker.docker_image_load:
+            path: "/opt/rita/rita-v2-{{  rita_version  }}-image.tar"
+          register: load_result
+          #This final one prints a list of the loaded images if we use the above 2 stanzas to load from a file.
+        - name: "RITA Install: Print loaded image names."
+          ansible.builtin.debug:
+            msg: "Loaded the following images: {{  load_result.image_names | join(', ')  }}"
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+    - name: "RITA Install: Transfer rita shell script to target system."
+      copy:
+        src: ./opt/rita.sh
+        dest: /usr/local/bin/rita
+        owner: root
+        group: root
+        mode: 0755
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+    - name: "RITA Install: Transfer rita install files to /opt/rita."
+      copy:
+        src: ./opt/
+        dest: /opt/rita
+        owner: root
+        group: root
+        mode: 0755
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+    - name: "RITA Install: Transfer rita user files to /etc/rita."
+      copy:
+        src: ./etc/
+        dest: /etc/rita
+        owner: root
+        group: root
+        mode: 0755
+      tags:
+        - docker
+        - rita
+        - linux
+        - linuxdeb
+        - linuxrpm
+
+#Late tasks, including rebooting
+    - name: "RITA Install: Check if reboot required on rpm-based systems."
+      command: needs-restarting -r
+      register: reboot_result
+      ignore_errors: true
+      when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' )
+      tags:
+        - packages
+        - linux
+        - linuxrpm
+
+    - name: "RITA Install: Check if reboot required on Debian/Ubuntu-based systems."
+      register: reboot_required_file
+      stat:
+        path: /var/run/reboot-required
+        get_checksum: no
+      when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Ubuntu' )
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+
+    - name: "RITA Install: Rebooting system if needed."
+      reboot:
+        reboot_timeout: 120
+      when: ( ansible_connection != 'local' and ( ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Ubuntu' ) and ( reboot_required_file.stat.exists ) ) or ( ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and ( reboot_result.rc == 1 ) ) )
+      register: reboot_status
+      async: 1
+      poll: 0
+      tags:
+        - packages
+        - linux
+        - linuxdeb
+        - linuxrpm

installer/install_scripts/sshprep

@@ -0,0 +1,315 @@
+#!/bin/bash
+#Performs all the setup steps needed to connect to one or more hosts listed on the command line
+#Copyright 2022 William Stearns <william.l.stearns@gmail.com>
+#Released under the GPL 3.0
+#Version 0.1.5
+
+
+
+askYN() {
+    # Prints a question mark, reads repeatedly until the user
+    # repsonds with t/T/y/Y or f/F/n/N.
+	TESTYN=""
+	while [ "$TESTYN" != 'Y' ] && [ "$TESTYN" != 'N' ] ; do
+		echo -n '? ' >&2
+		read -e TESTYN <&2 || :
+		case $TESTYN in
+		T*|t*|Y*|y*)		TESTYN='Y'	;;
+		F*|f*|N*|n*)		TESTYN='N'	;;
+		esac
+	done
+
+	if [ "$TESTYN" = 'Y' ]; then
+		return 0 #True
+	else
+		return 1 #False
+	fi
+}
+
+
+fail() {
+	echo "$@ , exiting." >&2
+	exit 1
+}
+
+
+require_util() {
+	#Returns true if all binaries listed as parameters exist somewhere in the path, False if one or more missing.
+	while [ -n "$1" ]; do
+		if ! type -path "$1" >/dev/null 2>/dev/null ; then
+			echo Missing utility "$1". Please install it. >&2
+			return 1	#False, app is not available.
+		fi
+		shift
+	done
+	return 0	#True, app is there.
+} #End of require_util
+
+
+status() {
+	if [ "$verbose" = 'yes' ]; then
+		echo "== $@" >&2
+	fi
+}
+
+
+step() {
+	if [ "$step" = 'yes' ]; then
+		echo "About to $@ , continue" >&2
+		if askYN ; then
+			return
+		else
+			exit 99
+		fi
+	fi
+}
+
+
+show_help() {
+	echo "This tool does all the setup needed to ssh to one or more hosts using ssh keys." >&2
+	echo "Usage:" >&2
+	echo >&2
+	echo "$0 [host] [host] [host]..." >&2
+	echo -e "\tDo basic setup and install ssh key on all hosts specified on the command line." >&2
+	echo -e "\tHost may optionally include a username, like bob@www.example.com" >&2
+	echo "$0 -v [host]..." >&2
+	echo -e "\tBe verbose - show steps to be taken." >&2
+	echo "$0 -s [host]..." >&2
+	echo -e "\tSingle step mode - ask permission before making changes (verbose mode is automatically enabled in single step mode)." >&2
+	echo "$0 --authpath /etc/ssh/keys-root/ [host]..." >&2
+	echo -e "\tLocation of the authorized_keys file (useful for VMWare ESXi).  This path is used for _all_ hosts on this command line." >&2
+	echo "$0 -h" >&2
+	echo "$0 --help" >&2
+	echo -e "\tShow this help text." >&2
+	exit 0
+}
+
+
+load_params() {
+	target_list=''
+
+	if [ "z$1" = "z" ]; then
+		echo "No target systems requested, this will perform basic ssh setup only." >&2
+	fi
+
+	while [ -n "$1" ]; do
+		case "$1" in
+		-v|--verbose)
+			verbose='yes'
+			;;
+		-s|--step)
+			verbose='yes'
+			step='yes'
+			;;
+		-h|--help)
+			show_help
+			;;
+		--authpath)
+			authpath="$2"
+			shift
+			;;
+		*)
+			target_list="$target_list $1"
+			;;
+		esac
+		shift
+	done
+}
+
+
+check_requirements() {
+	status "Checking system requirements"
+	require_util awk basename chmod cp date dig egrep head mkdir mktemp ssh ssh-add ssh-keygen touch uniq || fail "Missing one or more tools, please install and rerun"
+	step "perform system checks"
+	mkdir -p "$HOME/.ssh"
+	chmod -R go-rwx "$HOME/.ssh"
+	if [ -z "$SSH_AUTH_SOCK" ] || [ ! -S "$SSH_AUTH_SOCK" ]; then
+		fail 'We do not appear to have an ssh-agent to talk to.  You may want to add the following 3 lines to ~/.bashrc :
+if [ -z "$SSH_AUTH_SOCK" ] || [ ! -S "$SSH_AUTH_SOCK" ]; then
+	eval $(ssh-agent -s)
+fi
+	(and then logout, log back in, and try again.)'
+	fi
+	[ -w "$HOME/.ssh" ] || fail "User .ssh directory does not appear to be writeable"
+	[ -s "$HOME/.ssh/config" ] || touch "$HOME/.ssh/config"
+}
+
+
+check_key() {
+	status "Checking that we have a keypair available to load into the agent"
+	#Confirm we have an ssh RSA keypair; create if not.
+
+	if [ ! -s "$HOME/.ssh/id_rsa" ] && [ ! -s "$HOME/.ssh/id_rsa.pub" ]; then
+		status "No SSH RSA keypair, creating one.  We strongly encourage you to provide a strong passphrase."
+		step "create ssh rsa keypair"
+		ssh-keygen -t rsa -b 4096 -f "$HOME/.ssh/id_rsa"		#We don't force the comment; "user@host" is the default
+
+	elif [ -s "$HOME/.ssh/id_rsa" ] && [ ! -s "$HOME/.ssh/id_rsa.pub" ]; then
+		status "private key available but no public; we'll create the public key.  You will be asked for your id_rsa private key passphrase."
+		step "regenerate public key from private"
+		ssh-keygen -y -f "$HOME/.ssh/id_rsa" >"$HOME/.ssh/id_rsa.pub"
+	elif [ ! -s "$HOME/.ssh/id_rsa" ] && [ -s "$HOME/.ssh/id_rsa.pub" ]; then
+		status "Public key available, but no private"
+		fail "Missing private key, please provide $HOME/.ssh/id_rsa"
+	elif [ -s "$HOME/.ssh/id_rsa" ] && [ -s "$HOME/.ssh/id_rsa.pub" ]; then
+		status "Both private and public keys available"
+	fi
+}
+
+
+check_agent() {
+	status "Checking that keys are loaded into the agent"
+	#Confirm we have at least one key loaded into ssh-agent.
+
+	if [ $(ssh-add -l | wc -l) -eq 0 ] || [ "$(ssh-add -l)" = 'The agent has no identities.' ]; then
+		#No keys currently loaded into the agent, load at least one
+		[ -s "$HOME/.ssh/id_rsa" -a -s "$HOME/.ssh/id_rsa.pub" ] || check_key
+		status "About to load keys into ssh-agent; you may be asked for your passphrase(s)"
+		step "load keys into agent"
+		ssh-add
+	fi
+
+	if [ $(ssh-add -l | wc -l) -eq 0 ] || [ "$(ssh-add -l)" = 'The agent has no identities.' ]; then
+		fail "Unable to get any keys into the ssh-agent"
+	fi
+
+	status "ssh-agent has at least one key loaded, continuing"
+}
+
+
+prepend_config_block() {
+	#We prepend new stanzas since the end of the file may contain a default stanza like "Host * ......"
+
+	if [ -s "$HOME/.ssh/config" ]; then
+		cp -p "$HOME/.ssh/config" "$HOME/.ssh/config.$(date '+%Y%m%d%H%M%S%N')"
+	elif [ ! -e "$HOME/.ssh/config" ]; then
+		touch "$HOME/.ssh/config"
+	fi
+
+	new_config=$(mktemp -q /tmp/$(basename $0).XXXXXX) || fail "Unable to make temporary file during ~/.ssh/config update."
+	echo -e "$1" >"$new_config"
+	cat "$HOME/.ssh/config" >>"$new_config"
+	step "move new config file ($new_config) over existing $HOME/.ssh/config"
+	mv "$new_config" "$HOME/.ssh/config"
+}
+
+
+check_config_block() {
+	#Make sure we have a config block for this host, and add one if not
+	#Param 1: user, Param 2: host, Param 3: optional hostname or ip address to use as "Hostname"
+
+	status "Checking if we have a config block for $2"
+	if [ -n "$1" ]; then
+		user_line="	User			${1}"
+	else
+		user_line=''
+	fi
+
+	if egrep -q "(^Host.*[[:space:]]${2}[[:space:]]|^Host.*[[:space:]]${2}\$)" "$HOME/.ssh/config" ; then
+		status "$2 has a configuration block in ~/.ssh/config"
+	else
+		status "$2 does not have a configuration block in ~/.ssh/config, adding one."
+		if [ -n "$3" ]; then
+			status "User-supplied target IP or hostname ($3) for $2"
+			prepend_config_block "\nHost ${2} ${3}\n\tHostname\t\t${3}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+		elif [ -n "$(which getent 2>/dev/null)" ] && [ -n "$(getent hosts "$2")" ]; then
+			status "running on non-mac, appears we have an entry in /etc/hosts"
+			ip4=$(getent ahostsv4 "$2" | awk '{print $1}' | uniq | head -1)
+			ip6=$(getent ahostsv6 "$2" | awk '{print $1}' | uniq | head -1)
+
+			if [ -n "$ip4" -a -n "$ip6" ]; then
+				status "Have both ipv4 and ipv6 addresses for $2, adding both blocks"
+				prepend_config_block "\nHost ${2} ${ip4}\n\tHostname\t\t$ip4\n\tHostKeyAlias\t\t${2}\n${user_line}\n\nHost ${2}-v6 ${ip6}\n\tHostname\t\t$ip6\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+			elif [ -n "$ip4" -o -n "$ip6" ]; then		#One or the other but not both, which is why we can have both ip4 and ip6 on the Hostname line.
+				prepend_config_block "\nHost ${2} ${ip4}${ip6}\n\tHostname\t\t${ip4}${ip6}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+			#We don't include a block for neither ip4 nor ip6 set as "getent hosts" did successfully get one or the other.
+			fi
+		elif [ -n "$(which dscacheutil 2>/dev/null)" ] && [ -n "$(dscacheutil -q host -a name "$2")" ]; then
+			status "running on mac, appears we have an entry in /etc/hosts"
+			local_ip=$(dscacheutil -q host -a name "$2" | grep '^ip_address' | awk '{print $2}')
+			prepend_config_block "\nHost ${2} ${local_ip}\n\tHostname\t\t${local_ip}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+		else
+			status "No user-supplied target IP or hostname for $2, look one up"
+			ip4=$(dig +short ${2} A)
+			ip6=$(dig +short ${2} AAAA)
+
+			if [ -n "$ip4" ]; then
+				if [ -n "$ip6" ]; then
+					status "Have both ipv4 and ipv6 addresses for $2, adding both blocks"
+					prepend_config_block "\nHost ${2} ${ip4}\n\tHostname\t\t${ip4}\n\tHostKeyAlias\t\t${2}\n${user_line}\n\nHost ${2}-v6 ${ip6}\n\tHostname\t\t${ip6}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+				else
+					status "Have an IPv4 address only for $2"
+					prepend_config_block "\nHost ${2} ${ip4}\n\tHostname\t\t${ip4}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+				fi
+			else
+				if [ -n "$ip6" ]; then
+					status "Have an ipv6 address only for $2"
+					prepend_config_block "\nHost ${2} ${2}-v6 ${ip6}\n\tHostname\t\t${ip6}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+				else
+					status "No ipv4 or ipv6 address found for ${2}.  Please enter an IP address to use:"
+					read ip_or_hostname
+					prepend_config_block "\nHost ${2} ${ip_or_hostname}\n\tHostname\t\t${ip_or_hostname}\n\tHostKeyAlias\t\t${2}\n${user_line}\n"
+				fi
+			fi
+		fi
+	fi
+
+}
+
+
+sshprep_main() {
+	load_params "$@"
+	check_requirements
+	check_agent
+
+	status "Retrieving public key(s)"
+	pub_key_content="$(ssh-add -L)"
+
+	retval=0
+	for one_target in $target_list ; do
+		step "start setting up $one_target"
+		status "Setting up ssh connection to $one_target"
+		if echo "$one_target" | grep -q '@' - ; then
+			#We have an "@" in the target, so this is "user@host"
+			one_user="${one_target%%@*}"
+			one_host="${one_target##*@}"
+		else
+			#No "@" in target, so this is just a host
+			one_user=''
+			one_host="$one_target"
+		fi
+
+		#Check if we have a config block for this target; create one if not, place at top of file
+		check_config_block "$one_user" "$one_host"		#FIXME - arrange a way for the user to supply an optional hostname/ip as param3
+
+		status "Checking that we can now connect without using a password."
+		status "If this is your first time connecting to $one_target, you may be asked to verify a fingerprint.  To see the correct value, run 'ssh-keygen -l -f /etc/ssh/ssh_host_ecdsa_key.pub' on that system's console."
+		step "confirm $one_target reachable by key"
+		if ssh -o 'PasswordAuthentication=no' -o 'PreferredAuthentications=publickey' "$one_target" 'touch .ssh/reached-by-key ; true' 2>/dev/null ; then
+			status "Key appears to be successfully installed on $one_target"
+		else
+			status "Sending public key(s) over to $one_target .  You may be asked for your password for this account."
+			step "send public keys over to $one_target"
+			if [ -n "$authpath" ]; then
+				echo "$pub_key_content" \
+				 | ssh "$one_target" 'mkdir -p '"$authpath"' ; cat >>'"$authpath"'/authorized_keys ; chmod go-rwx '"$authpath"' '"$authpath"'/authorized_keys'
+			else
+				echo "$pub_key_content" \
+				 | ssh "$one_target" 'mkdir -p .ssh ; cat >>.ssh/authorized_keys ; chmod go-rwx ./ .ssh/ .ssh/authorized_keys'
+			fi
+
+			step "confirm $one_target reachable by key"
+			if ssh -o 'PasswordAuthentication=no' -o 'PreferredAuthentications=publickey' "$one_target" 'touch .ssh/reached-by-key ; true' 2>/dev/null ; then
+				status "Key is now successfully installed on $one_target"
+			else
+				status "Unable to reach $one_target by ssh key."
+				retval=2
+			fi
+		fi
+	done
+	exit "$retval"
+}
+
+
+sshprep_main "$@"
+

installer/rita-install.md

@@ -0,0 +1,66 @@
+
+#ansible install playbook for rita V2.
+
+#sample runs:
+#	Optional: Add the following block, without #'s  to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports).
+#The hosts must each be on their own line.  These can be full or short hostnames or a name following "Host" in ~/.ssh/config .
+#
+#[allritas]
+#ro810
+#ub2404
+#
+#	Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list:
+#
+#	ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '	#-C (no changes) means do a dry run
+#	ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: '
+
+
+
+#Many thanks to but-i-am-dominator for his help with this playbook.
+
+#Intended supported distributions.  These have had testing done on at least one version.
+#ADHD: 4 (based on ubuntu 20, works)
+#AlmaLinux: 8, 9 (tested: 9.4, works)
+#CentOS: stream 9 (tested: stream 9, works)
+#Debian: 11, 12 (tested: debian 12, works)
+#Fedora: 39, 40 (tested: fedora 40, works)
+#Kali: 2024.2 (tested: 2024.2, works)
+#OracleLinux: 9 (tested: 9.4, works.  NOTE: this was done on Security Onion 2.4.70 which is _based_ on Oracle Linux 9.4)
+#Rocky: 8, 9 (tested: rocky 8, works)
+#Security Onion: 2.4.70 (based on oracle linux 9, works)
+#Ubuntu 20.04, 22.04, 24.04 (tested: ubuntu 24.04, works)
+
+#We hope to support these in the future, but they are not supported at the moment.
+#MacOS: Sonoma
+#RHEL: 8, 9 (as of 20240618 there's a known conflict between rhel 9 and docker-ce:
+#Note: RHEL 9 is currently (20240618) broken with docker-ce (and docker knows this
+#and puts up a warning for this distro.  Current error from trying to install on rhel 9:
+#
+#fatal: [rhel9-aws]: FAILED! => {"changed": false, "failures": [],
+#"msg": "Depsolve Error occurred: \n Problem 1: cannot install the
+#best candidate for the job\n  - nothing provides container-selinux
+#>= 2:2.74 needed by docker-ce-3:26.1.4-1.el9.x86_64 from
+#docker-ce\n  - nothing provides iptables needed by
+#docker-ce-3:26.1.4-1.el9.x86_64 from docker-ce\n Problem 2: cannot
+#install the best candidate for the job\n  - nothing provides
+#container-selinux >= 2:2.74 needed by
+#containerd.io-1.6.33-3.1.el9.x86_64 from docker-ce", "rc": 1,
+#"results": []}
+
+#Intended supported CPU architectures - not all have been tested yet.  For any CPU architectures we hope to support, we need
+#to build rita for that architecture.  To confirm whether your CPU is 32 bit vs 64 bit, run
+#lshw | head | grep -i width
+#    width: 64 bits
+#
+#x86_64		#All testing so far has been on x86_64
+#Possible future supported architectures
+#aarch64	#Pi4 and Pi5, but note this requires a 64 bit OS like Ubuntu or recent RaspiOS64 for pi.  Appears to be equal to arm64.
+#armhf		#32 bit arm, likely includes pi3 and below (or pi4 and pi5 when running a 32 bit OS)
+#		#For reference, pi zero and pi1 are 32 bit/arm6hf, pi2 is 32 bit/armhf, and (64 bit) pi zero 2, pi3, and pi4 are arm64=aarch64 (though these may not have a 64 bit os to run on them.)
+
+
+
+
+
+
+

installer/run_dev.sh

@@ -0,0 +1,29 @@
+set -e
+droplet_ip="$1"
+if [ -z "$droplet_ip" ]; then
+    echo "droplet ip was not provided"
+    exit 1
+fi
+
+VERSION=$(git describe --always --abbrev=0 --tags)
+
+
+./generate_installer.sh
+tar -xf rita-${VERSION}.tar.gz
+./rita-${VERSION}-installer/install_rita.sh "root@$droplet_ip"
+
+
+# # # # ansible-playbook -i digitalocean_inventory.py -e "install_hosts=${droplet_ip}" "./rita-${VERSION}-installer/install_rita.yml"
+
+
+
+
+# copy over test data
+scp -r ../test_data/open_sni "root@$droplet_ip":/root/sample_logs
+
+# # copy over test script
+scp ./test_installed.sh "root@$droplet_ip":/root/test_installed.sh
+
+# run test script
+ssh -t "root@$droplet_ip" /root/test_installed.sh "$VERSION"
+# 

installer/test_installed.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -e
+
+VERSION="$1"
+
+if [ -z "$VERSION" ]; then 
+    echo "VERSION was not set"
+    exit 1
+fi
+
+# check that all files exist in expected locations
+[ -f /usr/local/bin/rita ] || { echo >&2 "rita should be in /usr/local/bin"; exit 1; }
+# opt files
+[ -f /opt/rita/rita.sh ] || { echo >&2 "rita.sh should be in /opt/rita"; exit 1; }
+[ -f /opt/rita/docker-compose.yml ] || { echo >&2 "docker-compose.yml should be in /opt/rita"; exit 1; }
+[ -f /opt/rita/.env ] || { echo >&2 ".env should be in /opt/rita"; exit 1; }
+
+# etc files
+[ -f /etc/rita/config.hjson ] || { echo >&2 "config.hjson should be in /etc/rita"; exit 1; }
+[ -f /etc/rita/config.xml ] || { echo >&2 "config.xml should be in /etc/rita"; exit 1; }
+[ -f /etc/rita/http_extensions_list.csv ] || { echo >&2 "http_extensions_list.csv should be in /etc/rita"; exit 1; }
+[ -f /etc/rita/logger-cron ] || { echo >&2 "logger-cron should be in /etc/rita"; exit 1; }
+[ -f /etc/rita/syslog-ng.conf ] || { echo >&2 "syslog-ng.conf should be in /etc/rita"; exit 1; }
+[ -f /etc/rita/timezone.xml ] || { echo >&2 "timezone.xml should be in /etc/rita"; exit 1; }
+[ -d /etc/rita/threat_intel_feeds ] || { echo >&2 "/threat_intel_feeds should be in /etc/rita"; exit 1; }
+
+# verify that sed worked during installer generation
+if [ "$(grep -c "image: ghcr.io/activecm/rita-v2:${VERSION}" /opt/rita/docker-compose.yml)" -ne 1 ]; then
+    echo "/opt/rita/docker-compose.yml should have ghcr.io/activecm/rita-v2:${VERSION} set as the image definition for the rita service."
+    exit 1
+fi 
+
+
+# verify .env has production looking values
+if [ "$(grep -c "^CONFIG_DIR=/etc/rita" /opt/rita/.env)" -ne 1 ]; then
+    echo "/opt/rita/.env should have CONFIG_DIR=/etc/rita set"
+    exit 1
+fi 
+
+if [ "$(grep -c "^CONFIG_FILE=/etc/rita/config.hjson" /opt/rita/.env)" -ne 1 ]; then
+    echo "/opt/rita/.env should have CONFIG_FILE=/etc/rita/config.hjson set"
+    exit 1
+fi 
+
+if [ "$(grep -c "^DB_ADDRESS=db:9000" /opt/rita/.env)" -ne 1 ]; then
+    echo "/opt/rita/.env should have DB_ADDRESS=db:9000 set"
+    exit 1
+fi 
+
+
+# verify rita version
+if [ "$(rita --version | grep -c "$VERSION")" -ne 1 ]; then
+    echo "rita version command did not work correctly"
+    exit 1
+fi 
+
+
+# verify rita can run an import
+rita import --database=testymcgee --logs=/root/sample_logs --rebuild
+
+# check to see if this database appears in db list
+rita list | grep "testymcgee"
+
+# check to see that this dataset has data in it
+rita view --stdout testymcgee

integration/beacons_test.go

@@ -0,0 +1,327 @@
+package integration_test
+
+import (
+	"activecm/rita/analysis"
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/util"
+	"context"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+// **** Beacon Integration Tests ****
+// These tests verify the main beacon score and the four sub-scores for a handful of known beacons.
+// They validate that the output of the beacon scoring functions are properly working together
+// to produce the desired score.
+func (it *ValidDatasetTestSuite) TestBeacons() { // used by valid dataset test suite
+
+	it.Run("Beacon Type Counts", func() {
+		t := it.T()
+		// check the total count for each beacon type
+		type countRes struct {
+			BeaconType string `ch:"beacon_type"`
+			Count      uint64 `ch:"count"`
+		}
+
+		cases := []countRes{
+			{BeaconType: "sni", Count: 3383},
+			{BeaconType: "ip", Count: 1252},
+		}
+
+		var res []countRes
+		// check total beacon count by beacon type
+		err := it.db.Conn.Select(it.db.GetContext(), &res, `
+		SELECT beacon_type, count() as count FROM threat_mixtape
+		WHERE beacon_score > 0
+		GROUP BY beacon_type
+		ORDER BY count DESC
+	`)
+		require.NoError(t, err)
+		require.EqualValues(t, res, cases)
+	})
+
+	// verify known beacon scores
+	it.Run("Verify Known Scores", func() {
+		t := it.T()
+
+		// these values can be validated by using ./get_beacon_info.py
+		beaconCases := []struct {
+			name          string
+			mixtapeResult analysis.ThreatMixtape
+		}{
+			{
+				name: "10.55.100.111 -> 165.227.216.194",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:        net.ParseIP("10.55.100.111"),
+						SrcNUID:    util.UnknownPrivateNetworkUUID,
+						Dst:        net.ParseIP("165.227.216.194"),
+						DstNUID:    util.PublicNetworkUUID,
+						Count:      20054,
+						BeaconType: "ip",
+					}, Beacon: analysis.Beacon{
+						Score:          1,
+						TimestampScore: 1,
+						DataSizeScore:  1,
+						DurationScore:  1,
+						HistogramScore: 1,
+					}},
+			},
+			{
+				name: "10.55.100.111 -> tile-service.weather.microsoft.com",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:     net.ParseIP("10.55.100.111"),
+						SrcNUID: util.UnknownPrivateNetworkUUID,
+						FQDN:    "tile-service.weather.microsoft.com",
+						DstNUID: util.PublicNetworkUUID,
+						Count:   48,
+					}, Beacon: analysis.Beacon{
+						BeaconType:     "sni",
+						Score:          1,
+						TimestampScore: 1,
+						DataSizeScore:  0.998,
+						DurationScore:  1,
+						HistogramScore: 1,
+					}},
+			},
+			{
+				name: "10.55.100.109 -> www.alexa.com",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:     net.ParseIP("10.55.100.109"),
+						SrcNUID: util.UnknownPrivateNetworkUUID,
+						FQDN:    "www.alexa.com",
+						Count:   607,
+					}, Beacon: analysis.Beacon{
+						BeaconType:     "sni",
+						Score:          0.896,
+						TimestampScore: 0.999,
+						DataSizeScore:  0.47,
+						DurationScore:  1,
+						HistogramScore: 0.937,
+					}},
+			},
+			{
+				name: "10.55.100.103 -> www.bankofamerica.com",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:     net.ParseIP("10.55.100.103"),
+						SrcNUID: util.UnknownPrivateNetworkUUID,
+						FQDN:    "www.bankofamerica.com",
+						Count:   24,
+					}, Beacon: analysis.Beacon{
+						BeaconType:     "sni",
+						Score:          0.465,
+						TimestampScore: 0.407,
+						DataSizeScore:  0.632,
+						DurationScore:  0.823,
+						HistogramScore: 0,
+					}},
+			},
+		}
+
+		for _, test := range beaconCases {
+			t.Run(test.name, func(t *testing.T) {
+				var hash util.FixedString
+				var err error
+
+				// set beacon value
+				beacon := test.mixtapeResult
+
+				// get the hash
+				if beacon.FQDN != "" {
+					hash, err = util.NewFixedStringHash(beacon.Src.String(), beacon.SrcNUID.String(), beacon.FQDN)
+					require.NoError(t, err)
+				} else {
+					hash, err = util.NewFixedStringHash(beacon.Src.String(), beacon.SrcNUID.String(), beacon.Dst.String(), beacon.DstNUID.String())
+					require.NoError(t, err)
+				}
+
+				// create a context with the hash parameter
+				ctx := clickhouse.Context(it.db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+					"hash": hash.Hex(),
+				}))
+				var res analysis.ThreatMixtape
+				err = it.db.Conn.QueryRow(ctx, `
+							SELECT src, src_nuid, dst, dst_nuid, fqdn, sum(count) as count, toFloat32(sum(beacon_score)) as beacon_score, toFloat32(sum(ts_score)) as ts_score, toFloat32(sum(ds_score)) as ds_score, toFloat32(sum(dur_score)) as dur_score, toFloat32(sum(hist_score)) as hist_score FROM threat_mixtape
+							WHERE hash = unhex({hash:String})
+							GROUP BY src, src_nuid, dst, dst_nuid, fqdn
+						`).ScanStruct(&res)
+				require.NoError(t, err)
+				require.EqualValues(t, beacon.Count, res.Count, "beacon connection count must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+				require.InDelta(t, beacon.Score, res.Score, 0.05, "beacon score must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+				require.InDelta(t, beacon.TimestampScore, res.TimestampScore, 0.05, "beacon timestamp score must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+				require.InDelta(t, beacon.DataSizeScore, res.DataSizeScore, 0.05, "beacon data size score must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+				require.InDelta(t, beacon.DurationScore, res.DurationScore, 0.05, "beacon duration score must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+				require.InDelta(t, beacon.HistogramScore, res.HistogramScore, 0.05, "beacon histogram score must match %s -> %s -> %s", beacon.Src.String(), beacon.Dst.String(), beacon.FQDN)
+			})
+		}
+	})
+
+	// verify that the strobe in this dataset is not reported as a beacon in the threat_mixtape table
+	it.Run("Strobe Not Reported As Beacon", func() {
+		t := it.T()
+		var count uint64
+		err := it.db.Conn.QueryRow(it.db.GetContext(), `
+				SELECT sum(count) FROM threat_mixtape
+				WHERE src = '192.168.88.2' AND dst = '165.227.88.15'
+				AND beacon_score > 0
+			`).Scan(&count)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, count, "known strobe 192.168.88.2 -> 165.227.88.15 should not exist in beacons")
+	})
+
+	// verify that there are no beacons with a connection count lower than the connection threshold
+	it.Run("Beacons Follow Connection Threshold", func() {
+		t := it.T()
+		var count uint64
+
+		ctx := clickhouse.Context(it.db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+			"threshold": strconv.Itoa(int(it.cfg.Scoring.Beacon.UniqueConnectionThreshold)),
+		}))
+		err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() FROM threat_mixtape
+					WHERE beacon_score > 0 AND ts_unique < {threshold:Int}
+				`).Scan(&count)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, count, "there should be no beacons with a unique timestamp count that is less than or equal to the connection threshold")
+
+	})
+
+}
+
+// Verify proxy beacons
+func TestProxyBeacons(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err, "loading config should not return an error")
+
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+
+	// import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/proxy", "test_proxy_beacons", false, false)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "test_proxy_beacons", cfg, nil)
+	require.NoError(t, err)
+
+	// check the total count for each beacon type
+	t.Run("Proxy Beacon Count", func(t *testing.T) {
+		var countRes uint64
+
+		// check total beacon count by beacon type
+		err = db.Conn.QueryRow(db.GetContext(), `
+				SELECT count(DISTINCT hash) as count FROM threat_mixtape
+				HAVING sum(proxy_count) > 0 AND sum(beacon_score) > 0
+			`).Scan(&countRes)
+		require.NoError(t, err)
+
+		require.EqualValues(t, 2, countRes, "there should be two beacons with proxy connections")
+	})
+
+	// verify known beacon scores
+	t.Run("Verify Known Scores", func(t *testing.T) {
+
+		// These values can be validated by using ./get_beacon_info.py
+		beaconCases := []struct {
+			name          string
+			mixtapeResult analysis.ThreatMixtape
+		}{
+			{
+				name: "10.136.0.18 -> www.honestimnotevil.com",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:              net.ParseIP("10.136.0.18"),
+						SrcNUID:          util.UnknownPrivateNetworkUUID,
+						FQDN:             "www.honestimnotevil.com",
+						Count:            357,
+						ProxyCount:       357,
+						TotalBytes:       963595,
+						TotalDuration:    12.454628000000003,
+						PortProtoService: []string{"3128:tcp:http,ssl"},
+					}, Beacon: analysis.Beacon{
+						BeaconType:     "sni",
+						Score:          0.979,
+						TimestampScore: 0.921,
+						DataSizeScore:  0.995,
+						DurationScore:  1,
+						HistogramScore: 1,
+					}},
+			},
+			{
+				name: "10.136.0.18 -> www.google.com",
+				mixtapeResult: analysis.ThreatMixtape{
+					AnalysisResult: analysis.AnalysisResult{
+						Src:              net.ParseIP("10.136.0.18"),
+						SrcNUID:          util.UnknownPrivateNetworkUUID,
+						FQDN:             "www.google.com",
+						Count:            6,
+						ProxyCount:       6,
+						TotalBytes:       139385,
+						TotalDuration:    0.725752,
+						PortProtoService: []string{"3128:tcp:http,ssl"},
+					}, Beacon: analysis.Beacon{
+						BeaconType:     "sni",
+						Score:          0.586,
+						TimestampScore: 0.48,
+						DataSizeScore:  0.865,
+						DurationScore:  1,
+						HistogramScore: 0,
+					}},
+			},
+		}
+
+		for _, test := range beaconCases {
+			t.Run(test.name, func(t *testing.T) {
+				// set beacon value
+				beacon := test.mixtapeResult
+
+				// get the hash
+				hash, err := util.NewFixedStringHash(beacon.Src.String(), beacon.SrcNUID.String(), beacon.FQDN)
+				require.NoError(t, err)
+
+				// create a context with the hash parameter
+				ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+					"hash": hash.Hex(),
+				}))
+
+				var res analysis.ThreatMixtape
+				err = db.Conn.QueryRow(ctx, `
+						SELECT src, src_nuid, dst, dst_nuid, fqdn, count, proxy_count, beacon_score, ts_score, ds_score, dur_score, hist_score FROM threat_mixtape
+						WHERE hash = unhex({hash:String}) AND count > 0
+					`).ScanStruct(&res)
+				require.NoError(t, err)
+
+				// verfiy basic proxy beacon requirements
+				require.Greater(t, res.ProxyCount, uint64(0), "proxy count must be greater than 0")
+				require.GreaterOrEqual(t, int64(res.Count), cfg.Scoring.Beacon.UniqueConnectionThreshold, "connection count must be greater than or equal to the connection threshold")
+
+				// verify scores
+				require.EqualValues(t, beacon.Count, res.AnalysisResult.Count, "connection count must match expected value")
+				require.EqualValues(t, beacon.ProxyCount, res.ProxyCount, "proxy count must match expected value")
+				require.InDelta(t, beacon.Score, res.Score, 0.05, "beacon score must match expected value")
+				require.InDelta(t, beacon.TimestampScore, res.TimestampScore, 0.05, "timestamp score must match expected value")
+				require.InDelta(t, beacon.DataSizeScore, res.DataSizeScore, 0.05, "data size score must match expected value")
+				require.InDelta(t, beacon.DurationScore, res.DurationScore, 0.05, "duration score must match expected value")
+				require.InDelta(t, beacon.HistogramScore, res.HistogramScore, 0.05, "histogram score must match expected value")
+
+			})
+		}
+
+	})
+
+}

integration/db_test.go

@@ -0,0 +1,108 @@
+package integration_test
+
+import (
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func (it *ValidDatasetTestSuite) TestPrimaryKeySize() {
+	t := it.T()
+
+	type tableRes struct {
+		Table               string  `ch:"table"`
+		NumParts            uint64  `ch:"num_parts"`
+		TotalMarks          uint64  `ch:"total_marks"`
+		AvgMarks            float64 `ch:"avg_marks"`
+		TotalPrimaryKeySize uint64  `ch:"total_primary_key_size"`
+		CompressionRatio    float64 `ch:"compression_ratio"`
+	}
+
+	/* NOTE: !!! PERFORMANCE-CRITICAL TEST !!!
+	The purpose of this test is to track the storage efficiency of the valid dataset across continuing development.
+	These values should only be increased (CompressionRatio decreased) in order to pass tests IF after deliberate consideration
+	has been made on whether or not there is no other performant way of storing the data needed.
+	Changes to the schema can cause performance impacts on data insertion and queries.
+	*/
+	allowedMaximums := map[string]tableRes{
+		"big_ol_histogram": {NumParts: 4, TotalMarks: 20, AvgMarks: 10, TotalPrimaryKeySize: 500, CompressionRatio: 0.6},
+		"conn":             {NumParts: 2, TotalMarks: 50, AvgMarks: 50, TotalPrimaryKeySize: 6000, CompressionRatio: 0.7},
+		"conn_tmp":         {NumParts: 2, TotalMarks: 50, AvgMarks: 50, TotalPrimaryKeySize: 6000, CompressionRatio: 0.7},
+		"dns":              {NumParts: 2, TotalMarks: 50, AvgMarks: 50, TotalPrimaryKeySize: 6000, CompressionRatio: 0.6},
+		"dns_tmp":          {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 100, CompressionRatio: 0.25},
+		"exploded_dns":     {NumParts: 2, TotalMarks: 50, AvgMarks: 50, TotalPrimaryKeySize: 2000, CompressionRatio: 0.4},
+		"http":             {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 600, CompressionRatio: 0.7},
+		"http_tmp":         {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 600, CompressionRatio: 0.7},
+		"http_proto":       {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 1000, CompressionRatio: 0.7},
+		"mime_type_uris":   {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 1000, CompressionRatio: 0.7},
+		"openconn":         {NumParts: 2, TotalMarks: 60, AvgMarks: 60, TotalPrimaryKeySize: 6000, CompressionRatio: 0.7},
+		"openconn_tmp":     {NumParts: 2, TotalMarks: 60, AvgMarks: 60, TotalPrimaryKeySize: 6000, CompressionRatio: 0.7},
+		"openhttp":         {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 700, CompressionRatio: 0.7},
+		"openhttp_tmp":     {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 700, CompressionRatio: 0.7},
+		"openssl":          {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 2000, CompressionRatio: 0.7},
+		"openssl_tmp":      {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 2000, CompressionRatio: 0.7},
+		"pdns":             {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 1000, CompressionRatio: 0.8},
+		"pdns_raw":         {NumParts: 2, TotalMarks: 30, AvgMarks: 30, TotalPrimaryKeySize: 4000, CompressionRatio: 0.8},
+		"port_info":        {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 400, CompressionRatio: 0.5},
+		"rare_signatures":  {NumParts: 4, TotalMarks: 10, AvgMarks: 5, TotalPrimaryKeySize: 400, CompressionRatio: 0.35},
+		"sniconn_tmp":      {NumParts: 2, TotalMarks: 40, AvgMarks: 40, TotalPrimaryKeySize: 600, CompressionRatio: 0.4},
+		"opensniconn_tmp":  {NumParts: 2, TotalMarks: 40, AvgMarks: 40, TotalPrimaryKeySize: 600, CompressionRatio: 0.4},
+		"openconnhash_tmp": {NumParts: 2, TotalMarks: 40, AvgMarks: 40, TotalPrimaryKeySize: 2000, CompressionRatio: 0.4},
+		"ssl":              {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 1500, CompressionRatio: 0.7},
+		"ssl_tmp":          {NumParts: 2, TotalMarks: 20, AvgMarks: 20, TotalPrimaryKeySize: 1500, CompressionRatio: 0.7},
+		"threat_mixtape":   {NumParts: 4, TotalMarks: 10, AvgMarks: 5, TotalPrimaryKeySize: 700, CompressionRatio: 0.75},
+		"tls_proto":        {NumParts: 2, TotalMarks: 10, AvgMarks: 10, TotalPrimaryKeySize: 600, CompressionRatio: 0.7},
+		"uconn":            {NumParts: 2, TotalMarks: 15, AvgMarks: 15, TotalPrimaryKeySize: 800, CompressionRatio: 0.5},
+		"uconn_tmp":        {NumParts: 2, TotalMarks: 60, AvgMarks: 60, TotalPrimaryKeySize: 2000, CompressionRatio: 0.35},
+		"udns":             {NumParts: 2, TotalMarks: 50, AvgMarks: 50, TotalPrimaryKeySize: 5000, CompressionRatio: 0.7},
+		"usni":             {NumParts: 4, TotalMarks: 10, AvgMarks: 5, TotalPrimaryKeySize: 700, CompressionRatio: 0.5},
+	}
+
+	// optimize tables before checking parts
+	for table := range allowedMaximums {
+		ctx := it.db.QueryParameters(clickhouse.Parameters{
+			"table": table,
+		})
+		err := it.db.Conn.Exec(ctx, `--sql
+			OPTIMIZE TABLE {table:Identifier} FINAL
+		`)
+		require.NoError(t, err)
+	}
+
+	var res []tableRes
+	ctx := it.db.QueryParameters(clickhouse.Parameters{
+		"database": it.db.GetSelectedDB(),
+	})
+	err := it.db.Conn.Select(ctx, &res, `--sql
+			SELECT 
+				table, 
+				count() AS num_parts, 
+				sum(marks) AS total_marks, 
+				avg(marks) AS avg_marks, 
+				sum(primary_key_bytes_in_memory) AS total_primary_key_size,  
+				-- sum(data_uncompressed_bytes) AS uncompressed_size,
+                -- sum(data_compressed_bytes) AS compressed_size, 
+				-- sum(data_compressed_bytes) / sum(data_uncompressed_bytes) AS compression_ratio 
+				1 - (1 / ( sum(data_uncompressed_bytes) / sum(data_compressed_bytes))) AS compression_ratio 
+			FROM system.parts
+			WHERE database = {database:String} AND active = 1
+			GROUP BY table
+		`)
+	require.NoError(t, err)
+	// require.Len(t, res, len(allowedMaximums), "there should be an equal number of tables for the sensor database")
+
+	for _, table := range res {
+		currTable := allowedMaximums[table.Table]
+		require.LessOrEqual(t, table.NumParts, currTable.NumParts, "%s should have max %d parts, got: %d", table.Table, currTable.NumParts, table.NumParts)
+		// only check the number of marks if there is more than one part,
+		// since having fewer, larger parts is more efficient because it reduces the overhead
+		// of managing many small files and their corresponding marks
+		if table.NumParts > 1 {
+			require.LessOrEqual(t, table.TotalMarks, currTable.TotalMarks, "%s should have max %d total marks, got: %d", table.Table, currTable.TotalMarks, table.TotalMarks)
+			require.LessOrEqual(t, table.AvgMarks, currTable.AvgMarks, "%s should have max %1.1f avg marks, got: %1.1f", table.Table, currTable.AvgMarks, table.AvgMarks)
+		}
+		require.LessOrEqual(t, table.TotalPrimaryKeySize, currTable.TotalPrimaryKeySize, "%s should have a max total in-memory primary key size of %d, got: %d", table.Table, currTable.TotalPrimaryKeySize, table.TotalPrimaryKeySize)
+		require.GreaterOrEqual(t, table.CompressionRatio, currTable.CompressionRatio, "%s should have a min compression ratio of %1.2f, got: %1.2f", table.Table, currTable.CompressionRatio*100, table.CompressionRatio*100)
+
+	}
+
+}

integration/filter_test.go

@@ -0,0 +1,715 @@
+package integration_test
+
+import (
+	"activecm/rita/analysis"
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"activecm/rita/progressbar"
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/charmbracelet/bubbles/progress"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"golang.org/x/sync/errgroup"
+)
+
+type FilterTestSuite struct {
+	suite.Suite
+	cfg *config.Config
+}
+
+func TestFilters(t *testing.T) {
+	suite.Run(t, new(FilterTestSuite))
+}
+
+// Reset config after each test since these tests load the config from a file
+func (it *FilterTestSuite) SetupSuite() {
+	t := it.T()
+	afs := afero.NewOsFs()
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	it.cfg = cfg
+}
+
+func (it *FilterTestSuite) SetupTest() {
+	t := it.T()
+	err := it.cfg.ResetConfig()
+	require.NoError(t, err)
+}
+
+func (it *FilterTestSuite) TearDownSuite() {
+	t := it.T()
+	err := it.cfg.ResetConfig()
+	require.NoError(t, err)
+}
+
+func (it *FilterTestSuite) TestNeverIncludeSubnets() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewMemMapFs()
+	afs2 := afero.NewOsFs()
+	err := afero.WriteFile(afs, "testsuite_config.hjson", []byte(`
+	{
+		filtering: {
+			filter_external_to_internal: true,
+			internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"],
+			never_included_subnets: ["10.55.100.0/24"],
+		},
+		threat_intel: {
+			online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+			custom_feeds_directory: "./deployment/threat_intel_feeds"
+		},
+		http_extensions_file_path: "../deployment/http_extensions_list.csv"
+	}
+	`), 0755)
+	require.NoError(t, err)
+
+	cfg, err := config.LoadConfig(afs, "testsuite_config.hjson")
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+	it.cfg = cfg
+	require.Contains(t, cfg.Filter.NeverIncludedSubnets, &net.IPNet{IP: net.IP{10, 55, 100, 0}, Mask: net.IPMask{255, 255, 255, 0}})
+
+	// // import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs2, "../test_data/valid_tsv", "never_include_subnet", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "never_include_subnet", cfg, nil)
+	require.NoError(t, err)
+
+	var count uint64
+
+	// verify that not all connections in 10.0.0.0/8 were filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM conn
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 14452-12584, count, "conn table should contain 1868 entries in 10.0.0.0/8, got: %d", count)
+
+	// 5531 in 10.55.100.0/24
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM ssl
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 5615-(5531), count, "ssl table should contain 63 entries in 10.0.0.0/8, got: %d", count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM http
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 2004-(1982), count, "http table should contain 22 entries in 10.0.0.0/8, got: %d", count)
+
+	// verify that all connections in 10.55.100.0/24 were filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM conn
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "conn table shouldn't contain any entries in 10.55.100.0/24")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "http table shouldn't contain any entries in 10.55.100.0/24")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "ssl table shouldn't contain any entries in 10.55.100.0/24")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "dns table shouldn't contain any entries in 10.55.100.0/24")
+
+}
+
+func (it *FilterTestSuite) TestNeverIncludeDomains() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewMemMapFs()
+	afs2 := afero.NewOsFs()
+	err := afero.WriteFile(afs, "testsuite_config2.hjson", []byte(`
+	{
+		filtering: {
+			filter_external_to_internal: true,
+			internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"],
+			never_included_domains: ["*.microsoft.com", "businessinsider.com"]
+		},
+		threat_intel: {
+			online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+			custom_feeds_directory: "./deployment/threat_intel_feeds"
+		},
+		http_extensions_file_path: "../deployment/http_extensions_list.csv"
+	}
+	`), 0755)
+	require.NoError(t, err)
+
+	cfg, err := config.LoadConfig(afs, "testsuite_config2.hjson")
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+
+	err = config.UpdateConfig(cfg)
+	it.cfg = cfg
+	require.NoError(t, err, "updating config should not return an error")
+
+	require.Contains(t, cfg.Filter.NeverIncludedDomains, "*.microsoft.com")
+	require.Contains(t, cfg.Filter.NeverIncludedDomains, "businessinsider.com")
+
+	// // import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs2, "../test_data/valid_tsv", "never_include_domain", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "never_include_domain", cfg, nil)
+	require.NoError(t, err)
+
+	var count uint64
+
+	// verify that all connections w/ fqdns ending in .microsoft.com are filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE cutToFirstSignificantSubdomain(host) = 'microsoft.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "http table shouldn't contain any entries with host ending in .microsoft.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE cutToFirstSignificantSubdomain(server_name) = 'microsoft.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "ssl table shouldn't contain any entries with server_name ending in .microsoft.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE cutToFirstSignificantSubdomain(query) = 'microsoft.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "dns table shouldn't contain any entries with query ending in .microsoft.com")
+
+	// verify that not all domains in .businessinsider.com are filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE cutToFirstSignificantSubdomain(query) = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "dns table should contain at least one entry with a query ending in .businessinsider.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE cutToFirstSignificantSubdomain(host) = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "http table should contain at least one entry with a host ending in .businessinsider.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE cutToFirstSignificantSubdomain(server_name) = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "ssl table should contain at least one entry with a server_name ending in .businessinsider.com")
+
+	// verify that businessinsider.com is filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE query = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "dns table shouldn't contain any entries with a query of .businessinsider.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE host = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "http table shouldn't contain any entries with a query of .businessinsider.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE server_name = 'businessinsider.com'
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "ssl table shouldn't contain any entries with a query of .businessinsider.com")
+
+}
+
+func (it *FilterTestSuite) TestAlwaysIncludeSubnets() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewMemMapFs()
+	afs2 := afero.NewOsFs()
+	err := afero.WriteFile(afs, "testsuite_config3.hjson", []byte(`
+	{
+		filtering: {
+			filter_external_to_internal: true,
+			internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"],
+			never_included_subnets: ["10.0.0.0/8"],
+			always_included_subnets: ["10.55.100.0/24"]
+		},
+		threat_intel: {
+			online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+			custom_feeds_directory: "./deployment/threat_intel_feeds"
+		},
+		http_extensions_file_path: "../deployment/http_extensions_list.csv"
+	}
+	`), 0755)
+	require.NoError(t, err)
+
+	cfg, err := config.LoadConfig(afs, "testsuite_config3.hjson")
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+
+	err = config.UpdateConfig(cfg)
+	it.cfg = cfg
+	require.NoError(t, err, "updating config should not return an error")
+
+	require.Contains(t, cfg.Filter.NeverIncludedSubnets, &net.IPNet{IP: net.IP{10, 0, 0, 0}, Mask: net.IPMask{255, 0, 0, 0}}, "never included subnets should contain 10.0.0.0/8")
+	require.Contains(t, cfg.Filter.AlwaysIncludedSubnets, &net.IPNet{IP: net.IP{10, 55, 100, 0}, Mask: net.IPMask{255, 255, 255, 0}}, "always included subnets should contain 10.55.100.0/24")
+
+	// // import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs2, "../test_data/valid_tsv", "always_include_subnet", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "always_include_subnet", cfg, nil)
+	require.NoError(t, err)
+
+	var count uint64
+
+	// verify that not all connections in 10.0.0.0/8 were filtered
+	conn := 12591
+	http := 1982
+	ssl := 5531
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM conn
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, conn, count, "conn table should contain %d entries in 10.0.0.0/8, got: %d", conn, count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM ssl
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, ssl, count, "ssl table should contain %d entries in 10.0.0.0/8, got: %d", ssl, count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM http
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.0.0.0/104') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.0.0.0/104')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, http, count, "http table should contain %d entries in 10.0.0.0/8, got: %d", http, count)
+
+	// verify that all connections in 10.55.100.0/24 were filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM conn
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, conn, count, "conn table shouldn contain %d any entries in 10.55.100.0/24, got: %d", conn, count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM http
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, http, count, "http table shouldn contain %d entries in 10.55.100.0/24, got: %d", http, count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM ssl
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, ssl, count, "ssl table should contain %d entries in 10.55.100.0/24", ssl, count)
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count(DISTINCT hash) FROM dns
+		WHERE isIPAddressInRange(IPv6NumToString(src), '::ffff:10.55.100.0/120') OR isIPAddressInRange(IPv6NumToString(dst), '::ffff:10.55.100.0/120')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 1, count, "dns table should contain 1 entry in 10.55.100.0/24")
+
+}
+
+func (it *FilterTestSuite) TestAlwaysIncludeDomains() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewMemMapFs()
+	afs2 := afero.NewOsFs()
+	err := afero.WriteFile(afs, "testsuite_config4.hjson", []byte(`
+	{
+		filtering: {
+			filter_external_to_internal: true,
+			internal_subnets: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"],
+			never_included_domains: ["*.microsoft.com", "businessinsider.com"],
+			always_included_domains: ["*.mp.microsoft.com", "analytics.businessinsider.com"]
+		},
+		threat_intel: {
+			online_feeds: ["https://feodotracker.abuse.ch/downloads/ipblocklist.txt"],
+			custom_feeds_directory: "./deployment/threat_intel_feeds"
+		},
+		http_extensions_file_path: "../deployment/http_extensions_list.csv"
+	}
+	`), 0755)
+	require.NoError(t, err)
+
+	cfg, err := config.LoadConfig(afs, "testsuite_config4.hjson")
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+
+	err = config.UpdateConfig(cfg)
+	it.cfg = cfg
+	require.NoError(t, err, "updating config should not return an error")
+
+	require.Contains(t, cfg.Filter.NeverIncludedDomains, "*.microsoft.com")
+	require.Contains(t, cfg.Filter.NeverIncludedDomains, "businessinsider.com")
+
+	require.Contains(t, cfg.Filter.AlwaysIncludedDomains, "*.mp.microsoft.com")
+	require.Contains(t, cfg.Filter.AlwaysIncludedDomains, "analytics.businessinsider.com")
+
+	// // import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs2, "../test_data/valid_tsv", "always_include_domain", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "always_include_domain", cfg, nil)
+	require.NoError(t, err)
+
+	var count uint64
+
+	// verify that all connections w/ fqdns ending in .microsoft.com (but not *.mp.microsoft.com) are filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE cutToFirstSignificantSubdomain(host) = 'microsoft.com' AND NOT endsWith(host, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "http table shouldn't contain any entries with host ending in .microsoft.com (but not *.mp.microsoft.com)")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE cutToFirstSignificantSubdomain(server_name) = 'microsoft.com' AND NOT endsWith(server_name, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "ssl table shouldn't contain any entries with server_name ending in .microsoft.com (but not *.mp.microsoft.com)")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE cutToFirstSignificantSubdomain(query) = 'microsoft.com' AND NOT endsWith(query, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, count, "dns table shouldn't contain any entries with query ending in .microsoft.com (but not *.mp.microsoft.com)")
+
+	// verify that connections w/ fqdns ending in *.mp.microsoft.com are not filtered
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM http
+		WHERE endsWith(host, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "http table should contain at least 1 entry with host ending in .mp.microsoft.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM ssl
+		WHERE endsWith(server_name, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "ssl table should contain at least 1 entry with server name ending in .microsoft.com")
+
+	err = db.Conn.QueryRow(db.GetContext(), `
+		SELECT count() FROM dns
+		WHERE endsWith(query, '.mp.microsoft.com')
+	`).Scan(&count)
+	require.NoError(t, err)
+	require.Greater(t, count, uint64(0), "dns table should contain at least 1 entry with query ending in .microsoft.com")
+
+	tables := []struct {
+		name  string
+		field string
+	}{{name: "dns", field: "query"}, {name: "http", field: "host"}, {name: "ssl", field: "server_name"}}
+
+	domains := []struct {
+		d                string
+		shouldBeFiltered bool
+	}{{d: "businessinsider.com", shouldBeFiltered: true}, {d: "static2.businessinsider.com", shouldBeFiltered: false}}
+
+	for _, domain := range domains {
+		for _, table := range tables {
+			chCtx := db.QueryParameters(clickhouse.Parameters{
+				"table":  table.name,
+				"field":  table.field,
+				"domain": domain.d,
+			})
+
+			// verify that not all subdomains in this domain are filtered (wildcard shouldn't apply)
+			err = db.Conn.QueryRow(chCtx, `
+				SELECT count() FROM {table:Identifier}
+				WHERE endsWith({field:Identifier}, {domain:String})
+			`).Scan(&count)
+			require.NoError(t, err)
+			require.Greater(t, count, uint64(0), "%s table should contain at least one entry with a domain that ends in .%s", table.name, domain.d)
+
+			// verify that domain is filtered (if it should be)
+			err = db.Conn.QueryRow(chCtx, `
+				SELECT count() FROM {table:Identifier}
+				WHERE {field:Identifier} = {domain:String}
+			`).Scan(&count)
+			require.NoError(t, err)
+			if domain.shouldBeFiltered {
+				require.EqualValues(t, 0, count, "%s table shouldn't contain any entries with a domain of %s", table, domain.d)
+			} else {
+				require.Greater(t, count, uint64(0), "%s table should contain at least one entry with a domain of %s", table.name, domain.d)
+			}
+
+		}
+	}
+}
+
+// TestFilterExternalToInternal also tests ICMP
+func (it *FilterTestSuite) TestFilterExternalToInternal() {
+	t := it.T()
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	cfg.Filter.FilterExternalToInternal = false
+	err = config.UpdateConfig(cfg)
+	it.cfg = cfg
+	require.NoError(t, err, "updating config should not return an error")
+
+	require.False(t, cfg.Filter.FilterExternalToInternal)
+
+	// // import data
+	importResults, err := cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/valid_tsv", "filter_ext_to_int", false, true)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "filter_ext_to_int", cfg, nil)
+	require.NoError(t, err)
+
+	// 	// there are ICMP connections that are only on connections that are external to internal
+
+	type protoInfo struct {
+		PortProtoService string `ch:"port_proto_service"`
+		ConnCount        uint64 `ch:"conn_count"`
+		BytesSent        int64  `ch:"bytes_sent"`
+		BytesReceived    int64  `ch:"bytes_received"`
+	}
+
+	type testData struct {
+		src          string
+		dst          string
+		portInfoList []protoInfo
+	}
+
+	testCases := []testData{
+		{
+			src: "165.227.88.15",
+			dst: "192.168.88.2",
+			portInfoList: []protoInfo{
+				{"icmp:3/3", 2, 4858, 0},
+			},
+		},
+		{
+			src: "66.218.84.141",
+			dst: "10.55.100.107",
+			portInfoList: []protoInfo{
+				{"icmp:3/3", 1, 240, 0},
+			},
+		},
+		{
+			src: "66.218.84.141",
+			dst: "10.55.100.104",
+			portInfoList: []protoInfo{
+				{"icmp:3/3", 1, 240, 0},
+			},
+		},
+		{
+			src: "66.218.84.141",
+			dst: "10.55.100.108",
+			portInfoList: []protoInfo{
+				{"icmp:3/3", 1, 240, 0},
+			},
+		},
+		{
+			src: "67.226.210.13",
+			dst: "10.55.100.106",
+			portInfoList: []protoInfo{
+				{"icmp:3/10", 1, 136, 0},
+			},
+		},
+		{
+			src: "67.226.210.15",
+			dst: "10.55.100.107",
+			portInfoList: []protoInfo{
+				{"icmp:3/10", 1, 136, 0},
+			},
+		},
+		{
+			src: "67.226.210.14",
+			dst: "10.55.100.108",
+			portInfoList: []protoInfo{
+				{"icmp:3/10", 1, 136, 0},
+			},
+		},
+		{
+			src: "24.220.6.168",
+			dst: "10.55.200.11",
+			portInfoList: []protoInfo{
+				{"icmp:3/13", 1, 96, 0},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		ctx := clickhouse.Context(context.Background(), clickhouse.WithParameters(clickhouse.Parameters{
+			"src": test.src,
+			"dst": test.dst,
+		}))
+
+		var res []protoInfo
+		err = db.Conn.Select(ctx, &res, `
+			SELECT concat(proto, ':', icmp_type, '/', icmp_code) AS port_proto_service,
+				countMerge(count) AS conn_count,
+				sumMerge(bytes_sent) AS bytes_sent,
+				sumMerge(bytes_received) AS bytes_received
+			FROM port_info
+			WHERE src={src:String} AND dst={dst:String}
+			GROUP BY src, dst, dst_port, proto, service, icmp_type, icmp_code
+		`)
+		require.NoError(t, err, "querying proto table should not produce an error")
+
+		// ensure that the length of the result list matches the expected value
+		require.Len(t, res, len(test.portInfoList), "length of result list should match expected value")
+
+		// ensure that the result list matches the expected value
+		require.ElementsMatch(t, test.portInfoList, res, "result list should match expected value")
+	}
+
+	/* ******* Mixtape Propagation *******
+	The entries that use ICMP don't have enough connections to qualify as beacons, so they don't appear in the mixtape.
+	In order to test the spagooper query that grabs the ICMP entries within port:proto:service, we have to go through
+	the results of the IP spagooper.
+	*/
+
+	// set up new analyzer
+	minTSBeacon, maxTSBeacon, notFromConn, err := db.GetBeaconMinMaxTimestamps()
+	require.NoError(t, err)
+	require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+	minTS, maxTS, notFromConn, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+	require.NoError(t, err)
+	require.False(t, notFromConn, "min and max timestamps should be from conn table")
+	require.False(t, useCurrentTime, "first seen analysis should not use the current time")
+
+	analyzer, err := analysis.NewAnalyzer(db, cfg, importResults.ImportID[0], minTS, maxTS, minTSBeacon, maxTSBeacon, useCurrentTime, false)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	queryGroup, ctx := errgroup.WithContext(ctx)
+
+	// create progress bars
+	bars := progressbar.New(ctx, []*progressbar.ProgressBar{
+		progressbar.NewBar("IP Connection Analysis ", 2, progress.New(progress.WithDefaultGradient())),
+	}, []progressbar.Spinner{})
+
+	type foundEntry struct {
+		PortProtoService string
+		ConnCount        uint64
+		TotalBytes       int64
+	}
+	type resData struct {
+		src          string
+		dst          string
+		portInfoList []foundEntry
+	}
+	var foundIPs []resData
+
+	var expectedResData []resData
+
+	for _, test := range testCases {
+		d := resData{
+			src: test.src,
+			dst: test.dst,
+		}
+		var p []foundEntry
+		for _, dd := range test.portInfoList {
+			p = append(p, foundEntry{
+				PortProtoService: dd.PortProtoService,
+				ConnCount:        dd.ConnCount,
+				TotalBytes:       (dd.BytesSent + dd.BytesReceived) * 2, // multiply by 2 for openconns
+			})
+		}
+		d.portInfoList = p
+		expectedResData = append(expectedResData, d)
+	}
+
+	queryGroup.Go(func() error {
+		for entry := range analyzer.UconnChan {
+
+			for _, test := range testCases {
+				if entry.Src.String() == test.src && entry.Dst.String() == test.dst {
+					f := resData{
+						src: entry.Src.String(),
+						dst: entry.Dst.String(),
+					}
+					var portProto []foundEntry
+					for _, p := range entry.PortProtoService {
+						port := foundEntry{
+							PortProtoService: p,
+							ConnCount:        entry.Count,
+							TotalBytes:       entry.TotalBytes,
+						}
+						portProto = append(portProto, port)
+					}
+					f.portInfoList = portProto
+					foundIPs = append(foundIPs, f)
+				}
+			}
+
+		}
+		return nil
+	})
+
+	queryGroup.Go(func() error {
+		err := analyzer.ScoopIPConns(ctx, bars)
+		require.NoError(t, err)
+		close(analyzer.UconnChan)
+		return err
+	})
+
+	queryGroup.Go(func() error {
+		_, err := bars.Run()
+		require.NoError(t, err)
+		return err
+	})
+
+	if err := queryGroup.Wait(); err != nil {
+
+		require.NoError(t, err)
+	}
+
+	require.ElementsMatch(t, expectedResData, foundIPs)
+}

integration/get-mime-type-uris.sh

@@ -0,0 +1,163 @@
+#!/opt/homebrew/bin/bash
+
+# Set the source IP and FQDN for testing
+# src='10.55.100.108'
+# fqdn='www.businessinsider.com'
+
+# src='10.55.100.107'
+# fqdn='static.adsafeprotected.com'	
+
+src='10.55.100.103'
+fqdn='ml314.com'
+
+
+# src='10.55.100.104'
+# fqdn='a.scorecardresearch.com'
+
+# src='10.55.100.106'
+# fqdn='www.alexa.com'
+
+# src='10.55.100.105'
+# fqdn='static1.businessinsider.com'
+
+directory=$(realpath ../test_data/valid_tsv)
+directory="$directory/"
+
+valid_mime_types_file=$(realpath ../deployment/http_extensions_list.csv)
+
+echo "------------------------------"
+echo "Source: $src"
+echo "FQDN: $fqdn"
+# echo "Log Directory: $directory"
+# echo "Valid MIME Types File: $valid_mime_types_file"
+echo "------------------------------"
+
+# set default field for the destination ip or fqdn in the log
+fqdn_field='9' # http host field
+zeek_uid_field='2' # field for the zeek uid in all log types (conn, http, ssl)
+src_field='3' # src IP field for all log types (conn, http, ssl)
+
+# set log fields
+port_field='6'
+proto_field='7'
+http_sig_field='13' # http useragent field
+method_field='8'
+uri_field='10'
+referrer_field='11'
+dst_mime_type_field='29'
+ts_field='1'
+import_time_field='4'
+
+declare -A valid_mime_types
+declare -A count
+declare -A mismatch_count
+declare -A mismatch_details
+
+
+while IFS= read -r line; do
+    mime_type=$(echo "$line" | csvcut -c 2)
+    extension=$(echo "$line" | csvcut -c 3)
+
+    # Remove quotes from mime_type and extension fields
+    mime_type=$(echo "$mime_type" | tr -d '"')
+    extension=$(echo "$extension" | tr -d '"')
+
+    if [[ -z "$mime_type" ]]; then
+        continue
+    fi
+    if [[ "$extension" == "none" ]]; then
+        extension=""
+    fi
+
+    # Handle multiple extensions
+    IFS=',' read -ra ext_array <<< "$extension"
+    for ext in "${ext_array[@]}"; do
+        ext=$(echo "$ext" | tr -d '.' | xargs) # Remove dots and trim whitespace
+        if [[ -n "$ext" ]]; then
+            valid_mime_types["$mime_type"]+="$ext,"
+        fi
+    done
+done < "$valid_mime_types_file"
+
+# debug output to check the valid MIME types
+# for key in "${!valid_mime_types[@]}"; do
+#     echo "Key: $key, Value: ${valid_mime_types[$key]}"
+# done
+# echo "------------------------------"
+
+
+get_http_info(){
+    awk -v src_field="$src_field" -v fqdn_field="$fqdn_field" -v src="$src" -v fqdn="$fqdn" -v uri_field="$uri_field" -v dst_mime_type_field="$dst_mime_type_field" -F'\t' '
+    ($src_field == src) && ($fqdn_field == fqdn) { print $uri_field "\t" $dst_mime_type_field }' "$directory""$1".log
+
+}
+
+
+get_extension() {
+    local path="$1"
+    # Remove the query string if present
+    path="${path%%\?*}"
+    # Check if the path does not contain a . or ends with a .
+    if [[ "$path" != *.* || "$path" == *"." ]]; then
+        echo ""
+    else
+        # Split the last segment by . and take the last element as the extension
+        echo "${path##*.}"
+    fi
+}
+
+process_http_file() {
+    local log_type="$1"
+    while IFS=$'\t' read -r uri dst_mime_types; do
+        if [[ -n "$uri" && "$uri" != '/' ]]; then
+
+            # extract the path from the URI, ignoring the query string
+            path=$(echo "$uri" | awk -F'?' '{print $1}')
+
+            # get the extension from the path
+            extension=$(get_extension "$path")
+
+            # echo "uri: $uri"
+            # echo "Path: $path"
+            # echo "Extension: $extension"
+
+            # check for mismatches for each MIME type in the array
+            IFS=',' read -ra mime_types_array <<< "$dst_mime_types"
+            for dst_mime_type in "${mime_types_array[@]}"; do
+                dst_mime_type=$(echo "$dst_mime_type" | xargs) # trim whitespace
+                if [[ -n "$dst_mime_type" ]]; then
+                    valid_extensions="${valid_mime_types[$dst_mime_type]}"
+                    if [[ -n "$valid_extensions" ]]; then
+                        IFS=',' read -ra ext_array <<< "$valid_extensions"
+                        if ! [[ " ${ext_array[@]} " =~ " ${extension} " ]]; then
+                            key="$uri|$path|$extension|$dst_mime_type"
+                            mismatch_count["$key"]=$((mismatch_count["$key"] + 1))
+                            if [[ -z "$extension" ]]; then
+                                mismatch_details["$key"]="URI: $uri\nPath: $path\nExtension: -\n"
+                            else
+                                mismatch_details["$key"]="URI: $uri\nPath: $path\nExtension: $extension\n"
+                            fi
+                            mismatch_details["$key"]+="dst_mime_type: $dst_mime_type\n"
+                            mismatch_details["$key"]+="valid extensions: ${valid_extensions}\n"
+                        fi
+                    fi
+                fi
+            done
+        fi
+    done < <(get_http_info "$log_type")
+}
+
+
+# run the process
+process_http_file "http"
+
+
+# print the results
+echo "Mismatch details and counts:"
+echo "------------------------------"
+for key in "${!mismatch_count[@]}"; do
+    echo -e "${mismatch_details[$key]}"
+    echo "Mismatch count: ${mismatch_count[$key]}"
+    echo "----"
+done
+

integration/get-port-info.sh

@@ -0,0 +1,217 @@
+#!/opt/homebrew/bin/bash
+
+# This script is used to gather data used in integration testing of the port_info table
+
+# src: the source IP
+# dst: the destination IP
+# directory: the directory where the logs are located
+
+directory=$(realpath ../test_data/valid_tsv)
+directory="$directory/"
+
+# src='10.55.100.111'
+# dst='24.220.113.59'
+
+# src='10.55.100.111'
+# dst='162.208.22.39'
+
+# src='10.55.100.103'
+# dst='code.jquery.com'
+
+# src='10.55.100.100'
+# dst='fls.doubleclick.net'
+
+
+# src='10.55.100.105'
+# dst='au.download.windowsupdate.com'
+
+# src='10.55.100.109'
+# dst='7.dl.delivery.mp.microsoft.com'
+
+
+# src='10.55.100.105'
+# dst='g.live.com'
+
+# src='10.55.100.107'
+# dst='sp.adbrn.com'
+
+# src='10.55.100.105'
+# dst='192.132.33.27'
+
+# src='10.55.200.10'
+# dst='217.70.179.1'
+
+# src='10.55.100.103'
+# dst='47.89.68.213'
+
+# src='10.55.100.111'
+# dst='34.209.114.116'
+
+# src='10.55.182.100'
+# dst='96.45.33.73'
+
+# # src='10.55.200.11'
+# # dst='205.251.198.178'
+
+# src='10.55.182.100'
+# dst='172.217.8.206'
+
+src='10.55.100.111'
+dst='165.227.216.194'
+
+method='ip' # options: 'ip' or 'fqdn'
+# method='fqdn'
+
+echo "Source: $src"
+if [ "$method" = 'ip' ]; then
+    echo "Destination: $dst"
+elif [ "$method" = 'fqdn' ]; then
+    echo "FQDN: $dst"
+fi
+
+echo "Method: $method"
+
+# set default field for the destination ip or fqdn in the log
+dst_field='5' # conn dst field
+http_dst_field='9' # http host field
+ssl_dst_field='10' # ssl server_name field
+
+zeek_uid_field='2' # field for the zeek uid in all log types (conn, http, ssl)
+
+# Set the field for the src IP in the logs
+src_field='3' # src IP field for all log types (conn, http, ssl)
+
+# Set the fields for the additional information in the logs
+port_field='6'
+proto_field='7'
+service_field='8'
+bytes_sent_field='18'
+bytes_received_field='20'
+
+# IFS=$'\n'  # Set the internal field separator to newline
+
+get_column() {
+    awk -v src_field="$src_field" -v dst_field="$dst_field" -v src="$src" -v dst="$dst" -v column="$1" -F'\t' '($src_field == src) && ($dst_field == dst) { print $column }' "$directory""$2".log
+}
+
+get_port_proto_services() {
+    awk -v src_field="$src_field" -v dst_field="$dst_field" -v src="$src" -v dst="$dst" -v port_field="$port_field" -v proto_field="$proto_field" -v service_field="$service_field" -v bytes_sent_field="$bytes_sent_field" -v bytes_received_field="$bytes_received_field" -F'\t' '
+    ($src_field == src) && ($dst_field == dst) { print $port_field ":" $proto_field ":" $service_field "\t" $bytes_sent_field "\t" $bytes_received_field }' "$directory""$1".log
+}
+
+get_zeek_uids() {
+    awk -v src_field="$src_field" -v dst_field="$2" -v src="$src" -v dst="$dst" -v uid_field="$zeek_uid_field" -F'\t' '($src_field == src) && ($dst_field == dst) { print $uid_field }' "$directory""$1".log
+} 
+
+get_port_proto_services_by_uid() {
+    awk -v src_field="$src_field" -v uid_field="$zeek_uid_field" -v src="$src" -v uid="$1" -v port_field="$port_field" -v proto_field="$proto_field" -v service_field="$service_field" -v bytes_sent_field="$bytes_sent_field" -v bytes_received_field="$bytes_received_field" -F'\t' '
+    ($src_field == src) && ($uid_field == uid) { print $port_field ":" $proto_field ":" $service_field "\t" $bytes_sent_field "\t" $bytes_received_field }' "$directory""$2".log
+}
+
+
+
+# Declare an associative array to hold the bytes sent and received for each tuple
+declare -A tuple_count
+declare -A bytes_sent
+declare -A bytes_received
+
+
+# Function to process each log file
+process_log_file() {
+    local log_type="$1"
+    while IFS=$'\t' read -r tuple sent received; do
+        if [[ -n "$tuple" && -n "$sent" && -n "$received" ]]; then
+            tuple_count["$tuple"]=$((tuple_count["$tuple"] + 1))
+            bytes_sent["$tuple"]=$((bytes_sent["$tuple"] + sent))
+            bytes_received["$tuple"]=$((bytes_received["$tuple"] + received))
+        fi
+    done < <(get_port_proto_services "$log_type")
+}
+
+process_log_file_by_uid() {
+    # echo "Processing UID: $1"
+    local uid="$1"
+    local log_type="$2"
+    while IFS=$'\t' read -r tuple sent received; do
+        if [[ -n "$tuple" && -n "$sent" && -n "$received" ]]; then
+            tuple_count["$tuple"]=$((tuple_count["$tuple"] + 1))
+            bytes_sent["$tuple"]=$((bytes_sent["$tuple"] + sent))
+            bytes_received["$tuple"]=$((bytes_received["$tuple"] + received))
+        fi
+    done < <(get_port_proto_services_by_uid "$uid" "$log_type")
+}
+
+
+if [ $method = 'ip' ]; then
+    # Process conn.log
+    process_log_file 'conn'
+    # Process open_conn.log
+    # process_log_file 'open_conn'
+elif [ "$method" = 'fqdn' ]; then
+    # Separate UIDs from http and ssl logs
+    http_uids=()
+    ssl_uids=()
+    openhttp_uids=()
+    openssl_uids=()
+
+    # get UIDs from http.log
+    while IFS= read -r uid; do
+        http_uids+=("$uid")
+    done < <(get_zeek_uids 'http' "$http_dst_field")
+    # debug: print length of UIDs
+    echo "Length of UIDs from http.log: ${#http_uids[@]}"
+
+    # get UIDs from ssl.log
+    while IFS= read -r uid; do
+        ssl_uids+=("$uid")
+    done < <(get_zeek_uids 'ssl' "$ssl_dst_field")
+    # debug: print length of UIDs
+    echo "Length of UIDs from ssl.log: ${#ssl_uids[@]}"
+
+    # # get UIDs from open_http.log
+    # while IFS= read -r uid; do
+    #     openhttp_uids+=("$uid")
+    # done < <(get_zeek_uids 'open_http' "$http_dst_field")
+    # # debug: print length of UIDs
+    # echo "Length of UIDs from open_http.log: ${#openhttp_uids[@]}"
+
+    # # get UIDs from open_ssl.log
+    # while IFS= read -r uid; do
+    #     openssl_uids+=("$uid")
+    # done < <(get_zeek_uids 'open_ssl' "$ssl_dst_field")
+    # # debug: print length of UIDs
+    # echo "Length of UIDs from open_ssl.log: ${#openssl_uids[@]}"
+
+    # Merge and remove duplicate UIDs
+    all_uids=("${http_uids[@]}" "${ssl_uids[@]}" ) # "${openhttp_uids[@]}" "${openssl_uids[@]}"
+    unique_uids=($(printf "%s\n" "${all_uids[@]}" | sort | uniq))
+
+    # print length of uids
+    echo "Length of All UIDs: ${#all_uids[@]}"
+    echo "Length of Unique UIDs: ${#unique_uids[@]}"
+
+    # Process conn.log and open_conn.log for each UID
+    for uid in "${unique_uids[@]}"; do
+        # echo "Processing UID: $id"  # debug: Print the UID being processed
+        process_log_file_by_uid "$uid" 'conn'
+        # process_log_file_by_uid "$uid" 'open_conn'
+    done
+
+    # Process conn.log for each unique Zeek UID
+    # for id in "${unique_uids[@]}"; do
+    #     awk -v id="$id" -F'\t' '($2 == id) { print $0 }' "$directory""conn.log" | while IFS=$'\t' read -r line; do
+    #         # Troubleshooting: print the entire matched line for the UID
+    #         echo "Matching line for UID $id: $line"
+    #     done
+    # done
+    
+fi
+
+# Output the unique tuples and their byte counts
+for tuple in "${!bytes_sent[@]}"; do
+    echo "Tuple: $tuple"
+    echo "Count: ${tuple_count[$tuple]}"
+    echo "Bytes Sent: ${bytes_sent[$tuple]}"
+    echo "Bytes Received: ${bytes_received[$tuple]}"
+    echo "----"
+done

integration/get-proto-info.sh

@@ -0,0 +1,211 @@
+#!/opt/homebrew/bin/bash
+
+# This script is used to gather data used in integration testing of the tls_proto and http_proto tables
+
+# src: the source IP
+# dst: the destination IP
+# directory: the directory where the logs are located
+
+directory=$(realpath ../test_data/valid_tsv)
+directory="$directory/"
+
+# src='10.55.100.111'
+# dst='24.220.113.59'
+
+# src='10.55.100.111'
+# dst='162.208.22.39'
+
+# src='10.55.100.103'
+# dst='code.jquery.com'
+
+# src='10.55.100.100'
+# dst='fls.doubleclick.net'
+
+
+# src='10.55.100.105'
+# dst='au.download.windowsupdate.com'
+
+# src='10.55.100.109'
+# dst='7.dl.delivery.mp.microsoft.com'
+
+
+# src='10.55.100.105'
+# dst='g.live.com'
+
+# src='10.55.100.107'
+# dst='sp.adbrn.com'
+
+# src='10.55.100.105'
+# dst='192.132.33.27'
+
+# src='10.55.200.10'
+# dst='217.70.179.1'
+
+# src='10.55.100.103'
+# dst='47.89.68.213'
+
+# src='10.55.100.111'
+# dst='34.209.114.116'
+
+# src='10.55.182.100'
+# dst='96.45.33.73'
+
+# src='10.55.100.110'	
+# fqdn='static-ssl.businessinsider.com'
+
+# src='10.55.100.104'
+# fqdn='cdn.taboola.com'
+
+# src='10.55.100.105'
+# fqdn='www.alexa.com'
+
+# 10.55.100.109	imasdk.googleapis.com
+# src='10.55.100.109'
+# fqdn='imasdk.googleapis.com'
+
+src='10.55.100.107'
+fqdn='ctldl.windowsupdate.com'
+
+# src='10.55.100.107'
+# fqdn='www.google.com'
+
+# src='10.55.100.111'
+# fqdn='ml314.com'
+
+# src='10.55.100.110'
+# fqdn='fe2.update.microsoft.com'
+
+# src='10.55.100.108'
+# fqdn='www.alexa.com'
+
+# src='10.55.100.106'
+# fqdn='settings-win.data.microsoft.com'
+
+# src='10.55.100.109'
+# fqdn='pixel.adsafeprotected.com'
+
+# src='10.55.100.100'
+# fqdn='oneclient.sfx.ms'
+
+# src='10.55.100.107'
+# fqdn='comet.yahoo.com'
+
+# src='10.55.100.107'
+# fqdn='www.googletagmanager.com'
+
+# src='10.55.100.110'
+# fqdn='www.facebook.com'
+
+# method='tls' # options: 'tls' or 'http'
+method='http'
+
+echo "----"
+echo "Source: $src"
+echo "FQDN: $fqdn"
+echo "Method: $method"
+echo "----"
+
+# set default field for the destination ip or fqdn in the log
+# set field for the fqdn in the logs
+fqdn_field='9' # http host field
+if [ "$method" = 'tls' ]; then 
+    fqdn_field='10'
+fi
+
+zeek_uid_field='2' # field for the zeek uid in all log types (conn, http, ssl)
+
+# Set the field for the src IP in the logs
+src_field='3' # src IP field for all log types (conn, http, ssl)
+
+# Set the fields for the additional information in the logs
+port_field='6'
+proto_field='7'
+
+# set field for the signature field in the logs
+http_sig_field='13' # http useragent field
+ssl_sig_field='22' # ssl ja3 field
+
+# set ssl-specific fields
+validation_status_field='21'
+version_field='7'
+
+# set http-specific fields
+method_field='8'
+uri_field='10'
+referrer_field='11'
+dst_mime_type_field='29'
+
+# IFS=$'\n'  # Set the internal field separator to newline
+
+
+get_tls_info() {
+    awk -v src_field="$src_field" -v fqdn_field="$fqdn_field" -v src="$src" -v fqdn="$fqdn" -v ssl_sig_field="$ssl_sig_field" -v version_field="$version_field" -v validation_status_field="$validation_status_field" -F'\t' '
+    ($src_field == src) && ($fqdn_field == fqdn) { print $ssl_sig_field ":" $version_field ":" $validation_status_field }' "$directory""$1".log
+}
+
+get_http_info(){
+    awk -v src_field="$src_field" -v fqdn_field="$fqdn_field" -v src="$src" -v fqdn="$fqdn" -v http_sig_field="$http_sig_field" -v method_field="$method_field" -v uri_field="$uri_field" -v referrer_field="$referrer_field" -v dst_mime_type_field="$dst_mime_type_field" -F'\t' '
+    ($src_field == src) && ($fqdn_field == fqdn) { print $http_sig_field " -- method: " $method_field " -- uri: " $uri_field " -- referrer: " $referrer_field "\t" $dst_mime_type_field }' "$directory""$1".log
+
+}
+
+
+declare -A count
+declare -A mime_types
+
+# Function to process each log file
+process_tls_file() {
+    local log_type="$1"
+    while IFS=$'\t' read -r tuple; do
+        if [[ -n "$tuple" ]]; then
+            count["$tuple"]=$((count["$tuple"] + 1))
+        fi
+    done < <(get_tls_info "$log_type")
+}
+
+process_http_file() {
+    local log_type="$1"
+    while IFS=$'\t' read -r tuple dst_mime_type; do
+        if [[ -n "$tuple" ]]; then
+            count["$tuple"]=$((count["$tuple"] + 1))
+            if [[ "$dst_mime_type" != "-" ]]; then
+                mime_types["$tuple,$dst_mime_type"]=1
+            fi
+        fi
+    done < <(get_http_info "$log_type")
+}
+
+# add unique mime type to the array
+add_unique_mime_type() {
+    local tuple="$1"
+    local mime_type="$2"
+    if [[ ! ",${mime_types[$tuple]}," == *",$mime_type,"* ]]; then
+        add_unique_mime_type "$tuple" "$dst_mime_type"
+    fi
+}
+
+if [ $method = 'tls' ]; then
+    process_tls_file 'ssl' # ssl.log
+    # process_tls_file 'open_ssl' # open_ssl.log
+elif [ "$method" = 'http' ]; then
+    process_http_file 'http' # http.log
+    # process_http_file 'open_http' # open_http.log
+fi
+
+# Output the unique tuples and their byte counts
+for tuple in "${!count[@]}"; do
+    echo "$tuple"
+    echo "Count: ${count[$tuple]}"
+    if [ $method = 'http' ]; then
+        # collect unique MIME types for this tuple
+        unique_mime_types=()
+        for key in "${!mime_types[@]}"; do
+            if [[ $key == "$tuple,"* ]]; then
+                mime_type="${key#*,}"
+                unique_mime_types+=("$mime_type")
+            fi
+        done
+        echo "Unique MIME Types: ${unique_mime_types[@]}"
+    fi
+    echo "----"
+done

integration/get-rare-sig-info.sh

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+# this script is used to gather signature information from a tsv zeek log
+
+# src: the source ip
+# pair: optional destination ip or fqdn
+# method: 'src' or 'pair' (use pair if you want to specify a destination ip or fqdn)
+# directory: the directory where the logs are located
+
+src='10.55.100.104' # '10.55.100.111'
+# src='192.168.88.2'
+# src='45.125.66.34'
+# src='10.55.100.111'
+# src='10.55.100.100'
+pair='www.alexa.com'
+# pair='52.44.164.170'
+method='pair' # options: 'pair' or 'src'
+directory=$(realpath ../test_data/valid_tsv)
+directory="$directory/"
+
+# set the field for the src ip in the logs
+src_field='3' # src ip field for all log types (conn, http, ssl) is 3
+
+# set default field for the pair (if specified, the destination ip or fqdn in the log)
+conn_pair_field='5' # conn dst field
+http_pair_field='9' # http host field (change to 5 if using dst ip as the pair)
+ssl_pair_field='10' # ssl server_name field (change to 5 if using dst ip as the pair)
+
+# set field for the dst ip in the logs
+dst_ip_field='5' # dst ip field for all log types (conn, http, ssl) is 5
+
+# set field for the fqdn in the logs
+http_fqdn_field='9' # http host field
+ssl_fqdn_field='10' # ssl server_name field
+
+# set field for the signature field in the logs
+http_sig_field='13' # http useragent field
+ssl_sig_field='22' # ssl ja3 field
+
+IFS=$'\n'  # set the internal field separator to newline
+
+get_column() {
+    if [ $method = 'src' ]; then
+        awk -v src_field="$src_field" -v src="$src" -v column="$2" -F'\t' '($src_field == src) { print $column }' "$directory""$1".log
+    elif [ $method = 'pair' ]; then
+        awk -v src_field="$src_field" -v src="$src" -v pair_field="$3" -v pair="$4" -v column="$2" -F'\t' '($src_field == src) && ($pair_field == pair) { print $column }' "$directory""$1".log
+    else
+        echo "Invalid method. Please choose 'src' or 'pair'."        
+    fi 
+}
+
+get_column_with_specific_sig() {
+    if [ $method = 'src' ]; then
+        awk -v src_field="$src_field" -v src="$src" -v column="$2" -v sig_field="$5" -v sig="$6" -F'\t' '($src_field == src) && ($sig_field == sig) { print $column }' "$directory""$1".log
+    elif [ $method = 'pair' ]; then
+        awk -v src_field="$src_field" -v src="$src" -v pair_field="$3" -v pair="$4" -v column="$2" -v sig_field="$5" -v sig="$6" -F'\t' '($src_field == src) && ($pair_field == pair) && ($sig_field == sig) { print $column }' "$directory""$1".log
+    else
+        echo "Invalid method. Please choose 'src' or 'pair'."        
+    fi 
+}
+
+get_missing_host_useragents(){
+    missing_host='-'
+    awk -v src_field="$src_field" -v src="$src" -v pair_field="$http_fqdn_field" -v pair="$missing_host" -v column="$http_sig_field" -F'\t' '($src_field == src) && ($pair_field == pair) { print $column }' "$directory""http.log"
+}
+
+get_missing_host_useragents_all_srcs(){
+    missing_host='-'
+    awk -v pair_field="$http_fqdn_field" -v pair="$missing_host" -v column="$http_sig_field" -F'\t' '($pair_field == pair) { print $column }' "$directory""http.log"
+}
+
+# get the useragents 
+unique_useragent_count=0
+useragent_list=()
+echo "-- Useragents:"
+for useragent in $( get_column 'http' $http_sig_field $http_pair_field $pair | sort | uniq); do
+    if [ "$useragent" != "-" ]; then
+
+        # get unique dst ip count for the given useragent
+        dst_ip_count=0
+        for dst_ip in $( get_column_with_specific_sig 'http' $dst_ip_field $http_pair_field $pair $http_sig_field $useragent | sort | uniq); do
+            if [ "$dst_ip" != "-" ]; then
+                dst_ip_count=$((dst_ip_count + 1))
+            fi
+        done
+
+        # get unique fqdn count for the given useragent
+        fqdn_count=0
+        for fqdn in $( get_column_with_specific_sig 'http' $http_fqdn_field $http_pair_field $pair $http_sig_field $useragent | sort | uniq); do
+            if [ "$fqdn" != "-" ]; then
+                fqdn_count=$((fqdn_count + 1))
+            fi
+        done
+
+        echo "$useragent" #" ~~~~~~ times_used_dst: " "$dst_ip_count" " ~~~~~~ times_used_fqdn: " "$fqdn_count"
+        unique_useragent_count=$((unique_useragent_count + 1))
+    fi
+done
+echo "-- Unique Useragent Count: $unique_useragent_count"
+
+# get the ja3 signatures
+unique_ja3_count=0
+ja3_list=()
+echo "-- JA3:"
+for ja3 in $( get_column 'ssl' $ssl_sig_field $ssl_pair_field $pair | sort | uniq); do
+    if [ "$ja3" != "-" ]; then
+
+        # get unique dst ip count for the given ja3
+        dst_ip_count=0
+        for dst_ip in $( get_column_with_specific_sig 'ssl' $dst_ip_field $ssl_pair_field $pair $ssl_sig_field $ja3 | sort | uniq); do
+            if [ "$dst_ip" != "-" ]; then
+                dst_ip_count=$((dst_ip_count + 1))
+            fi
+        done
+
+        # get unique fqdn count for the given ja3
+        fqdn_count=0
+        for fqdn in $( get_column_with_specific_sig 'ssl' $ssl_fqdn_field $ssl_pair_field $pair $ssl_sig_field $ja3 | sort | uniq); do
+            if [ "$fqdn" != "-" ]; then
+                fqdn_count=$((fqdn_count + 1))
+            fi
+        done
+
+        echo "$ja3" #" ~~~~~~ times_used_dst: " "$dst_ip_count" " ~~~~~~ times_used_fqdn: " "$fqdn_count"
+        unique_ja3_count=$((unique_ja3_count + 1))
+    fi
+done
+echo "-- Unique JA3 Count: $unique_ja3_count"
+
+# get the missing_host_useragent signatures
+unique_missing_host_useragent_count=0
+# missing_host_useragent_list=()
+echo "-- Missing Host Useragents For Source IP (pair value does not apply):"
+for missing_host_useragent in $( get_missing_host_useragents | sort | uniq); do
+    if [ "$missing_host_useragent" != "-" ]; then
+        echo "$missing_host_useragent"
+        unique_missing_host_useragent_count=$((unique_missing_host_useragent_count + 1))
+    fi
+done
+echo "-- Unique Missing Host Useragent Count: $unique_missing_host_useragent_count"
+
+# get the missing_host_useragent signatures for all srcs
+unique_missing_host_useragent_count_all_srcs=0
+# missing_host_useragent_list=()
+echo "-- Missing Host Useragents For All Source IPs in Log :"
+for missing_host_useragent in $( get_missing_host_useragents_all_srcs | sort | uniq); do
+    if [ "$missing_host_useragent" != "-" ]; then
+        echo "$missing_host_useragent"
+        unique_missing_host_useragent_count_all_srcs=$((unique_missing_host_useragent_count_all_srcs + 1))
+    fi
+done
+echo "-- Unique Missing Host Useragent Count For All Source IPs: $unique_missing_host_useragent_count_all_srcs"
+
+unset IFS  # reset the internal field separator to default

integration/get-uconn-info.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+# this script is used to gather connection information on a unique conn, http or ssl pair from a tsv zeek log
+
+# src: the source ip
+# dst: the destination ip or fqdn
+# type: the log type (conn, http, ssl)
+# directory: the directory where the logs are located
+
+src='192.168.88.2'  # '10.55.100.109'
+dst='165.227.88.15' # 'www.alexa.com'
+type='conn'
+directory=$(realpath ../test_data/valid_tsv)
+directory="$directory/"
+
+# set default field for the destination ip or fqdn in the log
+field='5' # conn dst field
+if [ $type = 'http' ]; then field='9'; fi
+if [ $type = 'ssl' ]; then field='10'; fi
+
+total_src_bytes=0
+total_dst_bytes=0
+total_src_ip_bytes=0
+total_dst_ip_bytes=0
+total_ip_bytes=0
+total_duration=0
+total_src_packets=0
+total_dst_packets=0
+conn_count=0
+ts_list_len=0
+src_ip_bytes_list_len=0
+
+get_conn_column() {
+    awk -v src="$src" -v dst="$dst" -v dst_field="$field" -v column="$1" -F'\t' '($3 == src) && ($dst_field == dst) { print $column }' "$directory""$type".log
+}
+
+# get connection information for unique http or ssl pair by linking associated conn records via zeek uid
+if [ $type = 'http' ] || [ $type = 'ssl' ]; then
+    for id in $(awk -v src="$src" -v dst="$dst" -v dst_field="$field" -F'\t' '($3 == src) && ($dst_field == dst) { print $2 }' "$directory""$type".log | sort | uniq); do
+        while read -r dur src_bytes dst_bytes src_packets src_ip_bytes dst_packets dst_ip_bytes; do
+            if [ "$dur" != "-" ]; then total_duration=$(echo "$total_duration + $dur" | bc ); fi
+            if [ "$src_bytes" != "-" ]; then total_src_bytes=$((total_src_bytes + src_bytes)); fi
+            if [ "$dst_bytes" != "-" ]; then total_dst_bytes=$((total_dst_bytes + dst_bytes)); fi
+            if [ "$src_ip_bytes" != "-" ]; then 
+                total_src_ip_bytes=$((total_src_ip_bytes + src_ip_bytes))
+                src_ip_bytes_list_len=$((src_ip_bytes_list_len + 1))
+            fi
+            if [ "$dst_ip_bytes" != "-" ]; then total_dst_ip_bytes=$((total_dst_ip_bytes + dst_ip_bytes)); fi
+            if [ "$src_packets" != "-" ]; then total_src_packets=$((total_src_packets + src_packets)); fi
+            if [ "$dst_packets" != "-" ]; then total_dst_packets=$((total_dst_packets + dst_packets)); fi
+        done < <(awk -v id="$id" -F'\t' '($2 == id) { print $9,$10,$11,$17,$18,$19,$20 }' "$directory""conn.log")
+        conn_count=$((conn_count + 1))
+    done
+fi
+
+
+# get connection information for unique conn pair
+if [ "$type" = 'conn' ]; then
+    while read -r dur src_bytes dst_bytes src_packets src_ip_bytes dst_packets dst_ip_bytes; do
+        if [ "$dur" != "-" ]; then total_duration=$(echo "$total_duration + $dur" | bc ); fi
+        if [ "$src_bytes" != "-" ]; then total_src_bytes=$((total_src_bytes + src_bytes)); fi
+        if [ "$dst_bytes" != "-" ]; then total_dst_bytes=$((total_dst_bytes + dst_bytes)); fi
+        if [ "$src_ip_bytes" != "-" ]; then 
+            total_src_ip_bytes=$((total_src_ip_bytes + src_ip_bytes))
+            src_ip_bytes_list_len=$((src_ip_bytes_list_len + 1))
+        fi
+        if [ "$dst_ip_bytes" != "-" ]; then total_dst_ip_bytes=$((total_dst_ip_bytes + dst_ip_bytes)); fi
+        if [ "$src_packets" != "-" ]; then total_src_packets=$((total_src_packets + src_packets)); fi
+        if [ "$dst_packets" != "-" ]; then total_dst_packets=$((total_dst_packets + dst_packets)); fi
+        conn_count=$((conn_count + 1))
+    done < <(awk -v src="$src" -v dst="$dst" -v dst_field="$field" -F'\t' '($3 == src) && ($dst_field == dst) { print $9,$10,$11,$17,$18,$19,$20 }' "$directory""$type".log)
+fi
+
+
+# print connection information
+echo "Source: $src"
+if [ "$type" = 'conn' ]; then echo "DST: $dst"; else echo "FQDN: $dst"; fi
+echo "Connection Count: $conn_count"
+echo "Total Duration: $total_duration"
+echo "Total Source Bytes: $total_src_bytes"
+echo "Total Resp Bytes: $total_dst_bytes"
+echo "Total Source IP Bytes: $total_src_ip_bytes"
+echo "Total Resp IP Bytes: $total_dst_ip_bytes"
+echo "Total IP Bytes: $(($total_src_ip_bytes + $total_dst_ip_bytes))"
+echo "Total Source Packets: $total_src_packets"
+echo "Total Resp Packets: $total_dst_packets"
+echo "Length of Src IP Bytes List: $src_ip_bytes_list_len"
+
+# get timestamps for when the connection was first and last seen
+echo "First Seen: $( get_conn_column '1' | sort | head -n1)"
+echo "Last Seen: $( get_conn_column '1' | sort | tail -n1)"
+
+# get number of unique timestamps for connection
+for entry in $( get_conn_column '1' | sort | uniq); do
+    ts_list_len=$((ts_list_len + 1))
+done
+echo "Length of Unique TS List: $ts_list_len"
+
+
+# get unique destination count for http and ssl logs
+if [ "$type" = 'http' ] || [ "$type" = 'ssl' ]; then
+    dst_count=0
+    for entry in $( get_conn_column '5' | sort | uniq); do
+        dst_count=$((dst_count + 1))
+    done
+    echo "Count of Unique Destinations: $dst_count"
+fi
+
+# if log type is http, get the useragents for given src-fqdn pair
+if [ "$type" = 'http' ]; then
+    IFS=$'\n'  # set the internal field separator to newline
+    unique_useragent_count=0
+    useragent_list=()
+    for useragent in $( get_conn_column '13' | sort | uniq); do
+        if [ "$useragent" != "-" ]; then
+            useragent_list+=$useragent
+            unique_useragent_count=$((unique_useragent_count + 1))
+        fi
+    done
+    echo "Unique Useragent Count: $unique_useragent_count"
+    # echo "Useragents: $useragent_list" # comment in if needed
+    unset IFS  # reset the internal field separator to default
+fi
+
+# if log type is http, get the uris for the given src-fqdn pair
+if [ "$type" = 'http' ]; then
+    IFS=$'\n'  # set the internal field separator to newline
+    unique_uri_count=0
+    uri_list=()
+    for uri in $( get_conn_column '10' | sort | uniq); do
+        if [ "$uri" != "-" ]; then
+            uri_list+="$uri"
+            unique_uri_count=$((unique_uri_count + 1))
+        fi
+    done
+    echo "Unique URI Count: $unique_uri_count"
+    # echo "URIs: $uri_list" # comment in if needed
+    unset IFS  # reset the internal field separator to default
+fi
+
+# get connection counts per hour
+echo "Counts Per Hour:"
+get_conn_column '1' | awk '{ input_epoch = $1; rounded_epoch = input_epoch - (input_epoch % 3600); print rounded_epoch}' | sort | uniq -c

integration/get_beacon_info.py

@@ -0,0 +1,677 @@
+import json
+import os
+import glob
+import gzip
+import re
+import math
+from pathlib import Path
+from datetime import datetime
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.dates import DateFormatter
+from scipy import stats
+
+
+#dnscat2
+# src= '10.55.200.11'
+# dst='205.251.197.77'
+
+# src='10.136.0.18'
+# dst='171.161.198.100'
+# dst=''
+# fqdn='www.honestimnotevil.com'
+
+src='10.55.100.103'
+dst=''
+fqdn='www.bankofamerica.com'
+
+
+logType='tsv'
+
+# src='10.55.100.109'
+# fqdn='www.alexa.com'
+# dst='165.227.216.194'
+
+# path = './test_data/proxy'
+path = '/Users/lisa/Desktop/go_ws/rita-v2/test_data/valid_tsv'
+# path = '/home/parallels/Desktop/chris/dnscat2-ja3-strobe-agent'
+order = ['ts','uid','id.orig_h','id.orig_p','id.resp_h','id.resp_p','proto','service','duration','orig_bytes','resp_bytes','conn_state','local_orig','local_resp','missed_bytes','history','orig_pkts','orig_ip_bytes','resp_pkts','resp_ip_bytes','tunnel_parents','agent_hostname','agent_uuid']
+# size = 'multiple'
+size = 'single'
+
+connOrder = ['ts','uid','id.orig_h','id.orig_p','id.resp_h','id.resp_p','proto','service','duration','orig_bytes','resp_bytes','conn_state','local_orig','local_resp','missed_bytes','history','orig_pkts','orig_ip_bytes','resp_pkts','resp_ip_bytes','tunnel_parents','agent_hostname','agent_uuid']
+httpOrder = ['ts','uid','id.orig_h','id.orig_p','id.resp_h','id.resp_p','trans_depth','method','host','uri','referrer','version','user_agent','request_body_len','response_body_len','status_code','status_msg','info_code','info_msg','tags','username','password','proxied','orig_fuids','orig_filenames','orig_mime_types','resp_fuids','resp_filenames','resp_mime_types']
+sslOrder = ['ts','uid','id.orig_h','id.orig_p','id.resp_h','id.resp_p','version','cipher','curve','server_name','resumed','last_alert','next_protocol','established','cert_chain_fuids','client_cert_chain_fuids','subject','issuer','client_subject','client_issuer','validation_status','ja3','ja3s']
+
+
+pattern = dst
+
+
+bimodalBucketSize = 0.05
+bimodalOutlierRemoval = 1
+bimodalMinHours = 11
+durMinHours = 6
+consistencyIdealHours = 12
+histModeSensitivity = 0.05
+
+def main():
+    # validate combinations of src / dst/ fqdn variables
+    if src == "":
+        print("src IP not defined")
+    if src != "" and dst != "" and fqdn != "":
+        print("src IP, dst IP, and FQDN cannot all be defined, supply either IP beacon or SNI beacon")
+    if dst == "" and fqdn == "":
+        print("please define either dst IP or FQDN")
+
+    # run beacon analysis
+    if dst != "":
+        tsData, dsData, minTsStartOfHour, minTs, maxTs = getIPBeacon()
+        analyze(tsData, dsData, minTsStartOfHour, minTs, maxTs)
+    else:
+        tsData, dsData, minTsStartOfHour, minTs, maxTs = getSNIBeacon()
+        analyze(tsData, dsData, minTsStartOfHour, minTs, maxTs)
+
+# get IP beacons from conn.log
+def getIPBeacon():
+    minTsStartOfHour, minTs, maxTs, uids = getConnData()
+
+    tsData = []
+    dsData = []
+    # filter conn records by src, dst, and min timestamp
+    for uid in uids:
+        entry = uids[uid]
+ 
+        # we filter by minTsStartOfHour instead of minTs because the data is stored in hour buckets aligned to the start of the hour
+        if entry['src']==src and entry['dst']==dst and entry['ts'] >= minTsStartOfHour:
+            tsData.append(entry['ts'])
+            dsData.append(entry['ds'])
+
+    return tsData, dsData, minTsStartOfHour, minTs, maxTs
+
+
+
+# read log files line by line
+def read_lines(filename, headerOrder=None):
+    print("reading:", filename)
+    def read_tsv_line(line, headerOrder):
+        # limit the number of lines returned to ones that match the source address
+        if re.search(src, line):
+            details = line.split("\t")
+            # split items by tab character based on the header order for this tsv file
+            details = [x.strip() for x in details]
+            structure = {key:value for key, value in zip(headerOrder, details)}
+            if structure['ts'] != '#close':
+                return structure
+            
+            
+    # read gzipped logs
+    if filename.endswith('.gz'):
+        with gzip.open(filename, 'rt', encoding='UTF-8') as f:
+            # read logs line by line as json
+            if logType == "json":
+                for line in f:
+                    yield json.loads(line)
+            else:
+                # read logs line by line from tsv, automatically skipping the header
+                for i in range(8):
+                    next(f)
+                for line in f.readlines():
+                    d = read_tsv_line(line, headerOrder)
+                    if d is not None:
+                        yield d
+                
+    else:
+        # read uncompressed logs
+        with open(filename, "r") as f:
+            # read logs line by line as json
+            if logType == "json":
+                for line in f:
+                    yield json.loads(line)
+            else:
+                # read logs line by line from tsv, automatically skipping the header
+                for i in range(8):
+                    next(f)
+                for line in f:
+                    d = read_tsv_line(line, headerOrder)
+                    if d is not None:
+                        yield d
+               
+           
+               
+
+# read UIDs for either the HTTP or SSL files
+def readSNIFile(file_names, headerOrderList, fqdnField, uids, dsData, tsData, minTsStartOfHour):
+    for filename in file_names:
+        for line in read_lines(filename, headerOrder=headerOrderList):
+            try:
+                if line['id.orig_h']==src and line[fqdnField]==fqdn:
+                    # check if the SNI record has a matching conn record, and check to see if it has been used before
+                    # we only include the timestamp/byte once per zeek uid
+                    if line['uid'] in uids and uids[line['uid']]['used'] == False:
+                            uids[line['uid']]['used'] = True
+                            # we filter by minTsStartOfHour instead of minTs because the data is stored in hour buckets aligned to the start of the hour
+                            if uids[line['uid']]['ts'] >= minTsStartOfHour:
+                                dsData.append(uids[line['uid']]['ds'])
+                                tsData.append(uids[line['uid']]['ts'])
+            except KeyError: 
+                pass
+
+# get SNI beacon data from http.log, ssl.log, and conn.log
+def getSNIBeacon():
+    tsData = []
+    dsData = []
+
+    if size == 'single':
+        # get min/max timestamp data and the UIDs seen in the conn logs
+        # this is done in one step so that we don't need to read the conn logs twice
+        minTsStartOfHour, minTs, maxTs, uids = getConnData()
+
+        httpFiles = glob.glob(os.path.join(path, "http*.log"))
+        httpFiles.extend(glob.glob(os.path.join(path, "http*.log.gz")))
+        readSNIFile(httpFiles, httpOrder, "host", uids, dsData, tsData, minTsStartOfHour)
+
+        sslFiles = glob.glob(os.path.join(path, "ssl*.log"))
+        sslFiles.extend(glob.glob(os.path.join(path, "ssl*.log.gz")))
+        readSNIFile(sslFiles, sslOrder, "server_name", uids, dsData, tsData, minTsStartOfHour)
+       
+    return tsData, dsData, minTsStartOfHour, minTs, maxTs
+
+def analyze(tsData, dsData, minTsStartOfHour, minTs, maxTs):
+    if len(tsData) == 0 or len(dsData) == 0:
+        print("could not find connection pair in logs, are you sure you're looking at the right dataset?")
+        exit(0)
+    start = minTs
+    end = maxTs
+     
+    tsLength = len(tsData) - 1
+    print(("connection count:").rjust(20), len(tsData))
+    print(("tsLength:").rjust(20), tsLength)
+    
+    # sort data
+    sortedTsData = sorted(tsData, key=float)
+    sortedDsData = sorted(dsData, key=int)
+
+    # print(sortedData)
+    dsSizes, dsCounts, dsMode, dsModeCount= calculateDistinctCounts(dsData)
+
+    # get TS Score
+    tsScore, tsSkew, tsMadm = getTsScore(sortedTsData, tsLength)
+    dsScore, dsSkew, dsMadm = getDsScore(sortedDsData, dsMode)
+    print(("TS SCORE:").rjust(20), tsScore)
+    print(("DS SCORE:").rjust(20), dsScore)
+
+
+    # get histogram score
+    bucketDivs, freqList, freqCount, totalBars, longestRun, histScore = getTsHistogramScore(start, end, sortedTsData)
+    coverage, consistency, durScore = getDurationScore(start, end, int(tsData[0]), int(tsData[len(tsData)-1]), totalBars, longestRun)
+    print(("HIST SCORE:").rjust(20), histScore)
+    print(("DUR SCORE:").rjust(20), durScore)
+
+    score = np.round(((tsScore*0.25)+ \
+                       (dsScore*0.25)+ \
+                       (durScore*0.25)+ \
+                       (histScore*0.25)) * 1000) / 1000
+    print(("THE SCORE:").rjust(20), score)
+
+    # plot
+    plotHistogram(sortedTsData, start, end)
+
+
+def getDurationScore(dmin, dmax, histMin, histMax, totalBars, longestConsecutiveRun):
+    if durMinHours < 1 or consistencyIdealHours < 1:
+        print("invalid duration score thresholds")
+        exit(1)
+
+    if durMinHours < 1:
+        print("bimodalMinHours is invalid")
+        exit(1)
+
+    if consistencyIdealHours < 1:
+        print("consistencyIdealHours is invalid")
+        exit(1)
+
+    if dmax <= dmin:
+        print("max is less than or equal to min")
+        exit(1)
+
+    if histMax <= histMin:
+        print("hist max is less than or equal to hist min")
+        exit(1)
+    
+    coverage, consistency, score = 0, 0, 0
+    if totalBars >= durMinHours:
+
+        coverage = math.ceil(((histMax - histMin) / (dmax - dmin))*1000)  / 1000
+        if coverage > 1:
+            coverage = 1
+
+        consistency = math.ceil((longestConsecutiveRun / consistencyIdealHours) * 1000) / 1000
+        if consistency > 1:
+            consistency = 1
+
+        score = max(coverage, consistency)
+    return coverage, consistency, score
+
+def getDsScore(data, mode):
+    dsSkew, dsSkewScore = calculateBowleySkewness(data)
+
+    dsMadm, dsMadmScore = calculateMedianAbsoluteDeviation(data, 0)
+
+    # dsSmallnessScore = max(1 - mode / 65535, 0)
+
+    dsScore = np.round((((dsSkewScore+dsMadmScore)/2.0)*1000)) / 1000
+
+    return dsScore, dsSkew, dsMadm
+
+def getTsHistogramScore(start, end, sortedData):
+    # get bucket list
+    # we currently look at a 24 hour period
+    bucketDivs = createBuckets(start, end, 24)
+
+    # use timestamps to get freqencies for buckets
+    freqList, freqCount, totalBars, longestRun = createHistogram(bucketDivs, sortedData)
+
+    print(("hist bucket divs:").rjust(20), bucketDivs)
+    print(("hist freq list:").rjust(20), freqList)
+    print(("hist freq count:").rjust(20), freqCount)
+   
+    #  calculate first potential score: coefficient of variation
+	#  coefficient of variation will help score histograms that have jitter in the number of
+	#  connections but where the overall graph would still look relatively flat and consistent
+	#  calculate coefficient of variation score
+    cvScore = calculateCoefficientOfVariationScore(freqList)
+   
+    bimodalFitScore = calculateBimodalFitScore(freqCount, totalBars)
+    
+    return bucketDivs, freqList, freqCount, totalBars, longestRun, max(cvScore, bimodalFitScore)
+   
+def calculateCoefficientOfVariationScore(freqList):
+    total = 0
+
+    for e in freqList:
+        total += e
+
+    freqMean = total / len(freqList)
+
+    sd = float(0)
+
+    for j in range(0,len(freqList),1):
+        sd += math.pow(freqList[j] - freqMean, 2)
+    
+    sd = np.sqrt(sd / len(freqList))
+   
+    cv = sd / np.abs(freqMean)
+
+    cvScore = float(0)
+    if cv > 1:
+        cvScore = 0
+    else:
+        cvScore = np.round((1-cv)*1000) / 1000
+    
+    # cvScore = np.ceil((1 - cv)*1000) / 1000
+    if cvScore > 1:
+        cvScore = 1
+    return cvScore
+
+def calculateBimodalFitScore(freqCount, totalBars):
+    bimodalFit = 0
+
+    if totalBars >= bimodalMinHours:
+        largest = 0
+        secondLargest = 0
+
+        for key, val in freqCount.items():
+            if val > largest:
+                secondLargest = largest
+                largest = val
+            elif val > secondLargest:
+                secondLargest = val
+            
+        bimodalFit = (largest + secondLargest) / max(totalBars-bimodalOutlierRemoval, 1)
+   
+    bimodalFitScore = np.ceil(bimodalFit * 1000) / 1000
+    if bimodalFitScore > 1:
+        bimodalFitScore = 1
+    return bimodalFitScore
+
+
+# createBuckets
+def createBuckets(start, stop, size):
+    # Set number of dividers. Since the dividers include the endpoints,
+    # number of dividers will be one more than the number of desired buckets
+    total = size + 1
+
+    # declare list
+    bucketDivs = [None]*(total)#make([]int64, total)
+
+    # calculate step size
+    step = math.floor((math.floor(stop) - math.floor(start)) / (total - 1))
+
+    # set first bucket value to min timestamp
+    bucketDivs[0] = start
+
+    # create evenly spaced timestamp buckets
+    for i in range(1,total,1):
+    # for i := int64(1); i < total; i++ 
+        bucketDivs[i] = start + (i * step)
+
+
+    # set first bucket value to max timestamp
+    bucketDivs[total-1] = math.floor(stop)
+
+    return bucketDivs
+
+# createHistogram
+def createHistogram(bucketDivs, sortedData):
+    i = 0
+    bucket = bucketDivs[i+1]
+    # calculate the number of connections that occurred within the time span represented
+    # by each bucket
+    freqList = [0]*(len(bucketDivs)-1)#make([]int, len(bucketDivs)-1)
+
+    # loop over sorted timestamp list
+    for entry in sortedData:
+        # increment if still in the current bucket
+        if entry < bucket:
+            freqList[i]+=1
+            continue
+
+        # find the next bucket this value will fall under
+        for j in range(i+1,len(bucketDivs)-1,1):
+        # for j := i + 1; j < len(bucketDivs)-1; j++ {
+            if entry < bucketDivs[j+1]:
+                i = j
+                bucket = bucketDivs[j+1]
+                break
+
+
+        # increment count
+        # this will also capture and increment for a situation where the final timestamp is
+        # equal to the final bucket
+        freqList[i]+=1
+
+    freqCount, totalBars, longestRun = getFrequencyCounts(freqList)
+    return freqList, freqCount, totalBars, longestRun
+
+
+def getFrequencyCounts(freqList):
+    # make a fequency count map to track how often each value in freqList appears
+    largestConnCount = 0
+    totalBars = 0
+
+    for item in freqList:
+        if item > 0:
+            totalBars+=1
+
+        if item > largestConnCount:
+            largestConnCount = item
+       
+            
+    freqCount = {}
+    bucketSize = np.ceil(largestConnCount * histModeSensitivity)
+
+    freqListLen = len(freqList)
+    longestRun = 0
+    currentRun = 0
+
+    for i in range(0, freqListLen*2, 1):
+        item = freqList[i%freqListLen]
+
+        if item > 0:
+            currentRun+=1
+        else:
+            if currentRun > longestRun:
+                longestRun = currentRun
+            currentRun = 0
+        
+        if i < freqListLen:
+            if item > 0:
+                bucket = int(np.floor(item/bucketSize)*bucketSize)
+
+                if bucket in freqCount:
+                    freqCount[bucket]+=1
+                else:
+                    freqCount[bucket] = 1
+    
+    if currentRun > longestRun:
+        longestRun = currentRun
+
+    if longestRun > freqListLen:
+        longestRun = freqListLen
+    return freqCount, totalBars, longestRun
+
+
+def calculateBowleySkewness(data):
+    inputLength = len(data)
+    if inputLength == 0:
+        print("input length for calculating quartiles is 0")
+        exit(1)
+    
+    cutoff1, cutoff2 = 0, 0
+
+    if inputLength % 2 == 0:
+        cutoff1 = int(inputLength / 2)
+        cutoff2 = int(inputLength / 2)
+    else:
+        cutoff1 = int((inputLength - 1) / 2)
+        cutoff2 = cutoff1 + 1
+
+    q1 = np.median(sorted(data[0:cutoff1]))
+    q2 = np.median(sorted(data))
+    q3 = np.median(sorted(data[cutoff2:]))
+
+    print("quartiles", q1, q2, q3)
+    
+    num = q1 + q3 - (2 * q2)
+
+    print("num", num)
+
+    den = q3 - q1
+
+    print("den", den)
+
+    skewness = 0
+
+    if den >= 10 and q2 != q1 and q2 != q3:
+        skewness = num / den
+
+    print("SKKKEWWWNEESSS", skewness)
+
+    score = 1 - np.abs(skewness)
+
+    print("skwwwwe score", score)
+    
+
+    return skewness, score
+    
+
+def calculateMedianAbsoluteDeviation(data, defaultScore):
+
+    if len(data) == 0:
+        print("input slice for madm must not be empty")
+        exit(1)
+
+    dataSorted = sorted(data)
+
+    median = np.median(dataSorted)
+
+    deviations = [None] * len(dataSorted)
+
+    for key, val in enumerate(dataSorted):
+        deviations[key] = np.abs(val - median)
+
+    mad = np.median(deviations)
+
+    score = defaultScore
+    if median >= 1:
+        score = (median - mad) / median
+
+    if score < 0 or math.isnan(score):
+        score = 0
+
+    return mad, score
+
+
+
+
+def getTsScore(sortedData, tsLength):
+
+    #find the delta times between the timestamps
+    deltaTimesFull = [None] * (len(sortedData) - 1)
+
+    for i in range(tsLength):
+        tempdiff = int(sortedData[i+1]) - int(sortedData[i])
+        deltaTimesFull[i] = float(tempdiff)
+        # if tempdiff != 0:
+        #     diff.append(tempdiff)
+
+    deltaTimesFull = sorted(deltaTimesFull)
+
+    # print(diff)
+ # get a list of the intervals found in the data,
+    # the number of times the interval was found,
+    # and the most occurring interval
+    intervals, intervalCounts, tsMode, tsModeCount = calculateDistinctCounts(deltaTimesFull)
+   
+    nonZeroIndex = 0
+ 
+    for i in range(0,len(deltaTimesFull),1):
+        if deltaTimesFull[i] > 0:
+            nonZeroIndex = i
+            break
+    
+    deltaTimes = deltaTimesFull[nonZeroIndex:]
+
+    tsSkew, tsSkewScore = calculateBowleySkewness(deltaTimes)
+    
+    tsMadm, tsMadmScore = calculateMedianAbsoluteDeviation(deltaTimes, 1)
+   
+    tsScore = np.ceil(((tsSkewScore+tsMadmScore)/2)*1000) / 1000
+    # diffLength = len(deltaTimesFull)
+    # # Store the range for human analysis
+    # tsIntervalRange = deltaTimesFull[diffLength - 1] - deltaTimesFull[0]
+
+   
+
+    print(("ts intervals:").rjust(20), intervals)
+    print(("ts interval counts:").rjust(20), intervalCounts)
+    # print(("ts interval range:").rjust(20), tsIntervalRange)
+    print(("ts mode:").rjust(20), tsMode)
+    print(("ts mode count:").rjust(20), tsModeCount)
+    print(("ts range:").rjust(20), deltaTimes[len(deltaTimes) - 1] - deltaTimes[0] )
+
+    return tsScore, tsSkew, tsMadm
+
+# createCountMap returns a distinct data array, data count array, the mode,
+# and the number of times the mode occurred
+def calculateDistinctCounts(sortedIn):
+    if len(sortedIn) < 2:
+        print("not enough items in sorted data to calculate distinct counts")
+        exit(1)
+
+    sortedIn = sorted(sortedIn)
+
+    distinctNumbers = []
+    countsMap = {}
+
+    lastNumber = sortedIn[0]
+    distinctNumbers.append(lastNumber)
+    if lastNumber in countsMap:
+        countsMap[lastNumber]+=1
+    else:
+        countsMap[lastNumber] = 1
+
+
+    for currentNumber in sortedIn[1:]:
+        if lastNumber != currentNumber:
+            distinctNumbers.append(currentNumber)
+        
+        if currentNumber in countsMap:
+            countsMap[currentNumber]+=1
+        else:
+            countsMap[currentNumber] = 1
+        lastNumber = currentNumber
+    
+    countsArray = [None] * len(distinctNumbers)
+    mode = distinctNumbers[0]
+    maxCount = countsMap[mode]
+
+
+    for i, number in enumerate(distinctNumbers):
+        count = countsMap[number]
+        countsArray[i] = count
+
+        if count > maxCount:
+            maxCount = count
+            mode = number
+        
+    return distinctNumbers, countsArray, mode, maxCount
+
+def getConnData():
+    uids = {}
+    maxTs = 0
+    g = glob.glob(os.path.join(path, "conn*.log.gz"))
+    g.extend(glob.glob(os.path.join(path, "conn*.log")))
+    allConnIshFiles = set(g)
+    gn = glob.glob(os.path.join(path, "conn*summary*.log.gz"))
+    gn.extend(glob.glob(os.path.join(path, "conn*summary*.log")))
+    notRealConnFiles = set(gn)
+    connFiles = allConnIshFiles - notRealConnFiles
+    for filename in connFiles:
+        for line in read_lines(filename, connOrder):
+            if float(line['ts']) > maxTs:
+                maxTs = float(line['ts'])
+
+            uids[line['uid']] = {"src": line['id.orig_h'], "dst": line['id.resp_h'], "ts": float(line['ts']), "ds":int(line['orig_ip_bytes']), "used": False}
+
+    # subtract 24 hours from the max timestamp
+    minTs = math.floor(maxTs) - 86400
+    # round min to start of hour
+    minRoundedToStartOfHour = math.floor(minTs / 3600) * 3600
+    return math.floor(minRoundedToStartOfHour), minTs, maxTs, uids
+
+def plotHistogram(sortedData, start, end):
+    size, scale = 1000, 10
+    hist = pd.Series(sortedData)
+
+    binz = np.linspace(start,end,25, endpoint=True)
+    hist.plot.hist(#grid=True,
+                    #    start=1647302276, stop=1647388791,
+                    bins=binz,
+                    #    bins=24,
+                    #    bins=[1647302276,1647319762,1647360727,1647388791],
+
+                    rwidth=0.9,
+                    color='#607c8e')
+    # plt.title('Connection Histogram')
+    plt.xlabel('Timestamp')
+    plt.ylabel('Conn Count')
+
+
+    plt.yticks([0,1,2])
+
+    xlabels=[]
+    for ts in binz:
+        # print(label)
+        xlabels.append("%s" % datetime.fromtimestamp(ts))
+
+    plt.xticks(binz, xlabels, rotation=90)
+
+    # print("hist std", hist.std())
+
+    plt.show()
+
+# Abs returns two's complement 64 bit absolute value
+def Abs(a):
+	# mask = a >> 63
+    mask = np.right_shift(a, 63)
+    a = a ^ mask
+    return a - mask
+
+# Round returns rounded int64
+def Round(f):
+	return np.int64(np.floor(f + .5))
+
+main()
+# getMinMax()

integration/importer_test.go

@@ -0,0 +1,813 @@
+package integration_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	i "activecm/rita/importer"
+	"context"
+	"time"
+
+	"reflect"
+
+	"testing"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+func TestValidTSV(t *testing.T) {
+	validTSVSuite := new(ValidDatasetTestSuite)
+
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	// validTSVSuite.SetupClickHouse(t)
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+
+	// connect to clickhouse server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	require.NoError(t, err, "connecting to server should not produce an error")
+	validTSVSuite.server = server
+
+	// // import data
+	results, err := cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/valid_tsv", "dnscat2_ja3_strobe", false, false)
+	require.NoError(t, err)
+	validTSVSuite.importResults = results
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "dnscat2_ja3_strobe", cfg, nil)
+	require.NoError(t, err)
+
+	// determine which max timestamp to use for relative time calculations
+	_, maxTimestamp, _, err := db.GetBeaconMinMaxTimestamps()
+	require.NoError(t, err)
+
+	validTSVSuite.maxTimestamp = maxTimestamp
+	validTSVSuite.db = db
+	validTSVSuite.cfg = cfg
+	suite.Run(t, validTSVSuite)
+}
+
+func TestValidJSON(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	validJSONSuite := new(ValidDatasetTestSuite)
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+
+	// // import data
+	results, err := cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/valid_json", "dnscat2_ja3_strobe_json", false, false)
+	require.NoError(t, err)
+	validJSONSuite.importResults = results
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "dnscat2_ja3_strobe_json", cfg, nil)
+	require.NoError(t, err)
+
+	// determine which max timestamp to use for relative time calculations
+	// maxTimestamp, _, useCurrentTime, err := db.GetFirstSeenTimestamp()
+	_, maxTimestamp, _, err := db.GetBeaconMinMaxTimestamps()
+	require.NoError(t, err)
+
+	validJSONSuite.maxTimestamp = maxTimestamp
+	validJSONSuite.db = db
+	validJSONSuite.cfg = cfg
+	suite.Run(t, validJSONSuite)
+}
+
+// testCounts verifies that the correct number of records of each type were written to the database
+func (it *ValidDatasetTestSuite) TestCounts() {
+	// t.Helper()
+	t := it.T()
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	type testCase struct {
+		table               string
+		expectedDBCount     int
+		expectedImportCount int
+		importResultCount   uint64
+		uniqField           string
+		msg                 string
+	}
+
+	// Verify raw log counts
+	rawLogTestCases := []testCase{
+		{
+			table:               "conn",
+			expectedDBCount:     387004 + 1023, // 1023 is the number of conn records with no host header
+			expectedImportCount: 387004 + 1023,
+			importResultCount:   it.importResults.Conn,
+			msg:                 "written conn record count matches imported record count",
+		},
+		{
+			table:               "openconn",
+			expectedDBCount:     387004 + 1023,
+			expectedImportCount: 387004 + 1023,
+			importResultCount:   it.importResults.Conn,
+			msg:                 "written openconn record count should be zero because they were closed",
+		},
+		{
+			table:               "http",
+			expectedDBCount:     26150 - 1023,
+			expectedImportCount: 26181,
+			importResultCount:   it.importResults.HTTP,
+			msg:                 "written http record count matches imported record count",
+		},
+		{
+			table:               "openhttp",
+			expectedDBCount:     26150 - 1023,
+			expectedImportCount: 26181,
+			importResultCount:   it.importResults.OpenHTTP,
+			msg:                 "written openhttp record count should be zero because they were closed",
+		},
+		{
+			table:               "ssl",
+			expectedDBCount:     86616,
+			expectedImportCount: 86616,
+			importResultCount:   it.importResults.SSL,
+			msg:                 "written ssl record count matches imported record count",
+		},
+		{
+			table:               "openssl",
+			expectedDBCount:     86616,
+			expectedImportCount: 86616,
+			importResultCount:   it.importResults.OpenSSL,
+			msg:                 "written openssl record count should be zero because they were closed",
+		},
+		{
+			table:               "dns",
+			expectedDBCount:     315622,
+			expectedImportCount: 315622,
+			importResultCount:   it.importResults.DNS,
+			msg:                 "written dns record count matches imported record count",
+		},
+		{
+			table: "pdns_raw",
+			/* get number of IPv4 addresses in answers field in dns log
+			cat dns.log | cut -f 10,22 | grep -v "^\-" | cut -f 2 | awk -F',' '{ for(i=1;i<=NF;i++) print $i }' \
+			| grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" | wc -l
+			*/
+			expectedDBCount:     208296,
+			expectedImportCount: 208296,
+			importResultCount:   it.importResults.PDNSRaw,
+			msg:                 "written pdns_raw record count matches imported record count",
+		},
+	}
+	// Verify materialized views have the right number of records in them
+	materializedViewTestCases := []testCase{
+		{
+			table:               "uconn",
+			expectedDBCount:     14756,
+			expectedImportCount: 14756,
+			importResultCount:   14756,
+			uniqField:           "hash",
+			msg:                 "uconn table has correct number of distinct hashes",
+		},
+		{
+			table:               "usni",
+			expectedDBCount:     6817,
+			expectedImportCount: 6817,
+			importResultCount:   6817,
+			uniqField:           "hash",
+			msg:                 "usni table has correct number of distinct hashes",
+		},
+		{
+			table:               "udns",
+			expectedDBCount:     78452,
+			expectedImportCount: 78452,
+			importResultCount:   78452,
+			uniqField:           "hash",
+			msg:                 "udns table has correct number of distinct hashes",
+		},
+		{
+			table:               "pdns",
+			expectedDBCount:     5671,
+			expectedImportCount: 5671,
+			importResultCount:   5671,
+			uniqField:           "hash",
+			msg:                 "pdns table has correct number of distinct hashes",
+		},
+		{
+			table:               "exploded_dns",
+			expectedDBCount:     67740,
+			expectedImportCount: 67740,
+			importResultCount:   67740,
+			uniqField:           "fqdn",
+			msg:                 "exploded_dns table has correct number of unique fqdns",
+		},
+	}
+
+	for _, test := range rawLogTestCases {
+		// Verify correct total import counts
+		require.EqualValues(t, test.expectedImportCount, test.importResultCount, "imported correct number of %s records, got:%d", test.table, test.importResultCount)
+
+		// Verify all parsed log records were written to database
+		ctx := it.db.QueryParameters(clickhouse.Parameters{
+			"table": test.table,
+		})
+		err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM {table:Identifier}
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, test.expectedDBCount, result.Count, test.msg)
+	}
+
+	for _, test := range materializedViewTestCases {
+		// Verify correct total import counts
+		require.EqualValues(t, test.expectedImportCount, test.importResultCount, "unique %s map has correct length, expected: %d, got: %d", test.table, test.expectedImportCount, test.importResultCount)
+
+		// Verify all parsed log records were written to database
+		ctx := it.db.QueryParameters(clickhouse.Parameters{
+			"table":  test.table,
+			"column": test.uniqField,
+		})
+		err := it.db.Conn.QueryRow(ctx, `
+				SELECT count(DISTINCT {column:Identifier}) as count FROM {table:Identifier}
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, test.expectedDBCount, result.Count, "%s, got: %d", test.msg, result.Count)
+	}
+
+}
+
+// TestHTTPandSSLLinking makes sure that there are no more than 20 records per zeek UID in the http and open http tables
+// and that the http, open http, ssl and openssl tables have duration and bytes data set in one record per zeek uid
+func (it *ValidDatasetTestSuite) TestHTTPandSSLLinking() {
+	t := it.T()
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// verify http linking wrote no more than 20 records with the same zeek uid
+	err := it.db.Conn.QueryRow(it.db.GetContext(), `
+			SELECT count() as count FROM (
+				SELECT zeek_uid, count() as num_with_same_zeek_uid FROM http
+				GROUP BY zeek_uid
+			) WHERE num_with_same_zeek_uid > 20
+		`).ScanStruct(&result)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, result.Count, "http should have no more than 20 records with the same zeek uid")
+
+	// verify openhttp linking wrote no more than 20 records with the same zeek uid
+	err = it.db.Conn.QueryRow(it.db.GetContext(), `
+			SELECT count() as count FROM (
+				SELECT zeek_uid, count() as num_with_same_zeek_uid FROM openhttp
+				GROUP BY zeek_uid
+			) WHERE num_with_same_zeek_uid > 20
+		`).ScanStruct(&result)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, result.Count, "openhttp should have no more than 20 records with the same zeek uid")
+
+	// create list of tables which must duration and bytes fields set once per zeek uid
+	durAndBytesTables := []string{"http", "openhttp", "ssl", "openssl"}
+
+	// verify that each table in list has duration and bytes fields set once per zeek uid
+	for _, table := range durAndBytesTables {
+		ctx := it.db.QueryParameters(clickhouse.Parameters{
+			"table": table,
+		})
+		err = it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM (
+					SELECT zeek_uid, count() as num_with_dur FROM {table:Identifier}
+					WHERE duration > 0 AND (src_ip_bytes > 0 OR src_bytes > 0) AND (dst_ip_bytes > 0 OR dst_bytes > 0)
+					GROUP BY zeek_uid
+				) WHERE num_with_dur != 1
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, (table + " must have duration and bytes fields set once per zeek uid"))
+	}
+
+}
+
+// TestTableFieldsThatCannotBeUnset makes sure that fields which cannot be unset -
+// (zeek_uid, hash, src, dst, src_nuid, dst_nuid, query and fqdn) are not unset
+func (it *ValidDatasetTestSuite) TestTableFieldsThatCannotBeUnset() {
+
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// ✅ HASH
+	it.T().Run("ValidateHash", func(t *testing.T) {
+		// list of tables which must have set hash fields
+		hashTables := []string{"conn", "uconn", "openconn", "http", "usni", "openhttp", "ssl", "openssl", "dns", "udns", "pdns_raw", "pdns", "tls_proto", "http_proto", "threat_mixtape"}
+
+		// verify that each table in list has no unset zeek uids
+		for _, table := range hashTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() AS count FROM {table:Identifier}
+					WHERE hash==toFixedString('',16) OR hex(hash)=='00000000000000000000000000000000'
+				`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset hash fields"))
+		}
+	})
+
+	// ✅ IMPORT_ID
+	it.T().Run("ValidateImportID", func(t *testing.T) {
+		// list of tables which must have set import_id fields
+		importIDTables := []string{"conn", "threat_mixtape"}
+
+		// verify that each table in list has no unset import_id fields
+		for _, table := range importIDTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() AS count FROM {table:Identifier}
+					WHERE import_id==toFixedString('',16) OR hex(import_id)=='00000000000000000000000000000000' OR import_id=='' OR import_id IS NULL
+				`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset import_id fields"))
+		}
+	})
+
+	// ✅ ZEEK_UID
+	it.T().Run("ValidateZeekUID", func(t *testing.T) {
+		// list of tables which must have set zeek uid fields
+		zeekUIDTables := []string{"conn", "openconn", "dns", "http", "openhttp", "ssl", "openssl"}
+
+		// verify that each table in list has no unset zeek uids
+		for _, table := range zeekUIDTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() AS count FROM {table:Identifier}
+					WHERE zeek_uid==toFixedString('',16) OR hex(zeek_uid)=='00000000000000000000000000000000'
+				`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset zeek uid fields"))
+		}
+	})
+
+	// ✅ TIMESTAMP
+	it.T().Run("ValidateTimestamp", func(t *testing.T) {
+		// list of tables which must have set ts field
+		tsTables := []string{"conn", "openconn", "http", "openhttp", "ssl", "openssl", "dns", "pdns_raw"}
+
+		// verify that each table in list has no unset ts fields
+		for _, table := range tsTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() AS count FROM {table:Identifier}
+				WHERE ts=='1970-01-01 00:00:00' OR ts=='2036-02-07 06:28:16' OR ts >='2106-02-07 06:28:15'
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset or invalid timestamp fields"))
+		}
+	})
+
+	// ✅ IMPORT TIME
+	it.T().Run("ValidateImportTime", func(t *testing.T) {
+		// list of tables which must have set ts field
+		tsTables := []string{"conn", "http", "ssl", "dns", "pdns_raw"}
+
+		// verify that each table in list has no unset ts fields
+		for _, table := range tsTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() AS count FROM {table:Identifier}
+					WHERE import_time=='1970-01-01 00:00:00' OR import_time=='2036-02-07 06:28:16' OR import_time >='2106-02-07 06:28:15'
+				`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset or invalid timestamp fields"))
+		}
+	})
+
+	// ✅ HOUR
+	it.T().Run("ValidateHour", func(t *testing.T) {
+		// list of tables which must have set hour fields
+		hourTables := []string{"uconn", "usni", "udns", "exploded_dns", "threat_mixtape", "mime_type_uris", "port_info", "tls_proto", "http_proto", "rare_signatures"}
+
+		// verify that each table in list has no unset hour fields
+		for _, table := range hourTables {
+			columnName := "hour"
+			if table == "threat_mixtape" {
+				columnName = "analyzed_at"
+			}
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table":  table,
+				"column": columnName,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() AS count FROM {table:Identifier}
+				WHERE {column:Identifier}=='1970-01-01 00:00:00' OR {column:Identifier}=='2036-02-07 06:28:16' OR {column:Identifier} >='2106-02-07 06:28:15'
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset or invalid hour fields"))
+		}
+	})
+
+	// ✅ IMPORT HOUR
+	it.T().Run("ValidateImportHour", func(t *testing.T) {
+		// list of tables which must have set hour fields
+		hourTables := []string{"uconn", "usni", "udns", "exploded_dns", "mime_type_uris", "port_info", "tls_proto", "http_proto", "rare_signatures"}
+
+		// verify that each table in list has no unset hour fields
+		for _, table := range hourTables {
+
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+			err := it.db.Conn.QueryRow(ctx, `
+					SELECT count() AS count FROM {table:Identifier}
+					WHERE import_hour=='1970-01-01 00:00:00' OR import_hour=='2036-02-07 06:28:16' OR import_hour >='2106-02-07 06:28:15'
+				`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset or invalid hour fields"))
+		}
+	})
+
+	// ✅ SRC / SRC_NUID
+	it.T().Run("ValidateSrc", func(t *testing.T) {
+		// list of tables which must have set src/src_nuid fields
+		srcTables := []string{"conn", "uconn", "openconn", "http", "usni", "openhttp", "ssl", "openssl", "dns", "udns", "pdns_raw", "pdns"}
+
+		// verify that each table in list has no unset src and src_nuid fields
+		for _, table := range srcTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+
+			// check src
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM {table:Identifier}
+				WHERE src == '::'
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset src fields"))
+
+			// check src_nuid
+			err = it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM {table:Identifier}
+				WHERE src_nuid == '00000000-0000-0000-0000-000000000000'
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset src_nuid fields"))
+		}
+	})
+
+	// ✅ DST / DST_NUID
+	it.T().Run("ValidateDst", func(t *testing.T) {
+		// list of tables which must have set dst/dst_nuid fields
+		dstTables := []string{"conn", "uconn", "openconn", "http", "usni", "openhttp", "ssl", "openssl", "dns", "udns", "pdns_raw", "pdns"}
+
+		// verify that each table in list has no unset dst and dst_nuid fields
+		for _, table := range dstTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+
+			// check dst
+			err := it.db.Conn.QueryRow(ctx, `
+						SELECT count() as count FROM {table:Identifier}
+						WHERE dst == '::'
+					`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset dst fields"))
+
+			// check dst_nuid
+			err = it.db.Conn.QueryRow(ctx, `
+						SELECT count() as count FROM {table:Identifier}
+						WHERE dst_nuid == '00000000-0000-0000-0000-000000000000'
+					`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset dst_nuid fields"))
+		}
+
+	})
+
+	// ✅  ICMP type / ICMP code
+	it.T().Run("ValidateICMPTypeCode", func(t *testing.T) {
+		// list of tables which must have set icmp_type & icmp_code fields
+		connTables := []string{"conn", "openconn"}
+
+		// verify that each table in list has no unset dst and dst_nuid fields
+		for _, table := range connTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+
+			// check non-icmp entries
+			err := it.db.Conn.QueryRow(ctx, `
+						SELECT count() as count FROM {table:Identifier}
+						WHERE proto != 'icmp' AND (icmp_type > -1 OR icmp_code > -1)
+					`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have icmp type or code greater than -1 when proto is not icmp"))
+		}
+
+	})
+
+	// ✅ QUERY
+	it.T().Run("ValidateQuery", func(t *testing.T) {
+		// list of tables which must have set query field
+		queryTables := []string{"dns", "pdns_raw"}
+
+		// verify that each table in list has no unset query fields
+		for _, table := range queryTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM {table:Identifier} WHERE query == ''
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, (table + " must not have unset query fields"))
+		}
+	})
+
+	// ✅ FQDN
+	it.T().Run("ValidateQuery", func(t *testing.T) {
+		// list of tables which must have set fqdn filed
+		fqdnTables := []string{"udns", "pdns", "exploded_dns"}
+
+		// verify that each table in list has no unset fqdn fields
+		for _, table := range fqdnTables {
+			ctx := it.db.QueryParameters(clickhouse.Parameters{
+				"table": table,
+			})
+
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM {table:Identifier} WHERE fqdn == ''
+			`).ScanStruct(&result)
+			require.NoError(t, err)
+			require.EqualValues(t, 0, result.Count, "%s must not have unset fqdn fields, got: %d", table, result.Count)
+		}
+	})
+
+}
+
+// TestMixtapeFields
+// go test -v ./integration -run TestValidTSV/TestMixtapeFields
+func (it *ValidDatasetTestSuite) TestMixtapeFields() {
+
+	table := "threat_mixtape"
+	ctx := it.db.QueryParameters(clickhouse.Parameters{
+		"table": table,
+	})
+
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// ✅ last_seen : ensure that every last seen date in the mixtape is valid (greater than 0 and less than epoch overflow)
+	it.T().Run("ValidateLastSeen", func(t *testing.T) {
+		// get count of mixtape records with invalid last_seen fields
+		err := it.db.Conn.QueryRow(ctx, `
+			SELECT count() as count FROM {table:Identifier}
+			WHERE last_seen=='1970-01-01 00:00:00' OR last_seen=='2036-02-07 06:28:16' OR last_seen >='2106-02-07 06:28:15'
+		`).ScanStruct(&result)
+
+		// ensure that there is no error
+		require.NoError(t, err)
+
+		// ensure that there are no records with invalid last_seen fields
+		require.EqualValues(t, 0, result.Count, "last_seen must not have unset or invalid timestamp fields")
+	})
+
+	// // verify that each table in list has no unset ts fields
+	// for _, table := range tsTables {
+	// 	ctx := it.db.QueryParameters(clickhouse.Parameters{
+	// 		"table": table,
+	// 	})
+	// 	err := it.db.Conn.QueryRow(ctx, `
+	// 		SELECT count() AS count FROM {table:Identifier}
+	// 		WHERE ts=='1970-01-01 00:00:00' OR ts=='2036-02-07 06:28:16' OR ts >='2106-02-07 06:28:15'
+	// 	`).ScanStruct(&result)
+	// 	require.NoError(t, err)
+	// 	require.EqualValues(t, 0, result.Count, (table + " must not have unset or invalid timestamp fields"))
+	// }
+}
+
+// TestTSVLogFieldParsing verifies that every required field in each tsv log type has parsed data in the database
+func TestTSVLogFieldParsing(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// get config
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	// update config with clickhouse connection
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	cfg.Filter.FilterExternalToInternal = false
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+
+	// import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/open_conns/open", "test_tsv_field_parsing", false, false)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "test_tsv_field_parsing", cfg, nil)
+	require.NoError(t, err)
+
+	// test tsv log field parsing
+	testLogFieldParsing(t, db, reflect.TypeOf(i.ConnEntry{}), "conn")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.ConnEntry{}), "openconn")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.HTTPEntry{}), "http")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.HTTPEntry{}), "openhttp")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.SSLEntry{}), "ssl")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.SSLEntry{}), "openssl")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.DNSEntry{}), "dns")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.DNSEntry{}), "pdns_raw")
+}
+
+// TestJSONLogFieldParsing verifies that every required field in each json log type has parsed data in the database
+func TestJSONLogFieldParsing(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// load config
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	// update config with clickhouse connection
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not return an error")
+
+	// import data
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, "../test_data/json_with_all_fields", "json_with_all_fields", false, false)
+	require.NoError(t, err)
+
+	// connect to database
+	db, err := database.ConnectToDB(context.Background(), "json_with_all_fields", cfg, nil)
+	require.NoError(t, err)
+
+	// test json log field parsing
+	testLogFieldParsing(t, db, reflect.TypeOf(i.ConnEntry{}), "conn")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.ConnEntry{}), "openconn")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.HTTPEntry{}), "http")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.HTTPEntry{}), "openhttp")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.SSLEntry{}), "ssl")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.SSLEntry{}), "openssl")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.DNSEntry{}), "dns")
+	testLogFieldParsing(t, db, reflect.TypeOf(i.DNSEntry{}), "pdns_raw")
+
+}
+
+// testLogFieldParsing determines which fields a log entry needs to have and verifies that they are parsed in the database
+func testLogFieldParsing(t *testing.T, db *database.DB, log reflect.Type, table string) {
+	t.Helper()
+
+	// get list of string fields in log entry
+	var stringFields []string
+	for i := 0; i < log.NumField(); i++ {
+		if log.Field(i).Type.Kind() == reflect.String {
+			stringFields = append(stringFields, log.Field(i).Tag.Get("ch"))
+		}
+	}
+
+	// verify string type fields
+	checkLogTypeStringFields(t, db, table, stringFields)
+
+	// get list of number fields in log entry
+	var numFields []string
+	for i := 0; i < log.NumField(); i++ {
+		if typeIsNumber(log.Field(i).Type.Kind()) {
+			numFields = append(numFields, log.Field(i).Tag.Get("ch"))
+		}
+	}
+
+	// verify number type fields
+	checkLogTypeNumFields(t, db, table, numFields)
+
+	// TODO: verify array fields
+
+}
+
+// checkLogTypeStringFields verifies that every string field in a log type has parsed data in the database
+func checkLogTypeStringFields(t *testing.T, db *database.DB, table string, stringFields []string) {
+	t.Helper()
+
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// verify string type fields
+	for _, field := range stringFields {
+		// skip this field since it is populated after parsing
+		if field == "missing_host_useragent" {
+			continue
+		}
+		ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+			"table":  table,
+			"column": field,
+		}))
+
+		err := db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM (
+					SELECT count() as num_set FROM {table:Identifier} where {column:Identifier} != ''
+				) WHERE num_set == 0
+		  `).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "%v must contain some records with a parsed %v field", table, field)
+
+		err = db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM (
+					SELECT count() as num_set FROM {table:Identifier} where {column:Identifier} != ''
+				) WHERE num_set == 0
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "%v must contain some records with a parsed %v field", table, field)
+	}
+}
+
+// checkLogTypeNumFields verifies that every number field in a log type has parsed data in the database
+func checkLogTypeNumFields(t *testing.T, db *database.DB, table string, numFields []string) {
+	t.Helper()
+
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// verify string type fields
+	for _, field := range numFields {
+		// skip these fields since they are populated after parsing
+		if field == "filtered" || field == "missing_host_header" || field == "dst_local" ||
+			field == "icmp_type" || field == "icmp_code" {
+			continue
+		}
+
+		// skip these fields since they're not filled out in the test data
+		if field == "resumed" || field == "recursion_desired" || field == "recursion_available" {
+			continue
+		}
+
+		if field == "rejected" && table == "pdns_raw" {
+			continue
+		}
+
+		ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+			"table":  table,
+			"column": field,
+		}))
+
+		err := db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM (
+					SELECT count() as num_set FROM {table:Identifier} where {column:Identifier} > 0
+				) WHERE num_set == 0
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "%v must contain some records with a parsed %v field", table, field)
+
+		err = db.Conn.QueryRow(ctx, `
+				SELECT count() as count FROM (
+					SELECT count() as num_set FROM {table:Identifier} where {column:Identifier} > 0
+				) WHERE num_set == 0
+			`).ScanStruct(&result)
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "%v must contain some records with a parsed %v field", table, field)
+	}
+}
+
+// typeIsNumber determines if a type is a number
+func typeIsNumber(varType reflect.Kind) bool {
+	switch varType {
+	case reflect.Bool:
+		fallthrough
+	case reflect.Uint16:
+		fallthrough
+	case reflect.Int:
+		fallthrough
+	case reflect.Int32:
+		fallthrough
+	case reflect.Int64:
+		fallthrough
+	case reflect.Float32:
+		fallthrough
+	case reflect.Float64:
+		return true
+	default:
+		return false
+	}
+
+}
+
+// add tests for filtering

integration/integration_test.go

@@ -0,0 +1,108 @@
+package integration_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/joho/godotenv"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/clickhouse"
+)
+
+const ConfigPath = "./test_config.hjson"
+
+const TestDataPath = "../test_data"
+
+type (
+	DockerInfo struct {
+		clickhouseContainer  *clickhouse.ClickHouseContainer
+		clickhouseConnection string
+	}
+	ValidDatasetTestSuite struct {
+		suite.Suite
+		server       *database.ServerConn
+		db           *database.DB
+		cfg          *config.Config
+		maxTimestamp time.Time
+		minTimestamp time.Time
+		// useCurrentTime bool
+		importResults cmd.ImportResults
+	}
+
+	IntegrationTestSuite ValidDatasetTestSuite
+)
+
+var dockerInfo DockerInfo
+
+func TestMain(m *testing.M) {
+	// err := godotenv.Load("../.env")
+	if err := godotenv.Overload("../.env", "./test.env"); err != nil {
+		log.Fatal("Error loading .env file")
+	}
+	err := SetupClickHouse(&dockerInfo)
+	if err != nil {
+		log.Fatal(err)
+	}
+	os.Exit(m.Run())
+}
+
+// SetupClickHouse creates a ClickHouse container using the test.docker-compose.yml and handles taking it down when complete
+func SetupClickHouse(d *DockerInfo) error {
+	version := os.Getenv("CLICKHOUSE_VERSION")
+	if version == "" {
+		return errors.New("CLICKHOUSE_VERSION environment variable not set")
+	}
+	ctx := context.Background()
+	clickHouseContainer, err := clickhouse.RunContainer(ctx,
+		testcontainers.WithImage(fmt.Sprintf("clickhouse/clickhouse-server:%s-alpine", version)),
+		clickhouse.WithUsername("default"),
+		clickhouse.WithPassword(""),
+		clickhouse.WithDatabase("default"),
+		clickhouse.WithConfigFile(filepath.Join("../deployment/", "config.xml")),
+	)
+	if err != nil {
+		return err
+	}
+
+	d.clickhouseContainer = clickHouseContainer
+	connectionHost, err := clickHouseContainer.ConnectionHost(ctx)
+	if err != nil {
+		return err
+	}
+	d.clickhouseConnection = connectionHost
+
+	return nil
+}
+
+func VerifyNonRollingFiles(t *testing.T, logDir string) cmd.HourlyZeekLogs {
+	t.Helper()
+
+	fs := afero.NewOsFs()
+	// get hourly map of all log files in directory
+	// hourlyLogMap, _, err := cmd.GetHourlyLogMap(fs, logDir)
+	hourlyLogMap, _, err := cmd.WalkFiles(fs, logDir)
+	require.NoError(t, err)
+
+	// ensure that only the first hour contains logs
+	for hour := range hourlyLogMap[0] {
+		if hour == 0 {
+			require.NotEmpty(t, hourlyLogMap[hour], "first hour should contain logs")
+		} else {
+			require.Empty(t, hourlyLogMap[hour], "hours other than zero hour shouldn't contain logs")
+		}
+	}
+
+	return hourlyLogMap[0]
+}

integration/materialized_view_test.go

@@ -0,0 +1,807 @@
+package integration_test
+
+import (
+	"activecm/rita/importer"
+	"database/sql"
+	"errors"
+	"testing"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/stretchr/testify/require"
+)
+
+// testExplodedDNS tests the exploded_dns table fields and values
+func (it *ValidDatasetTestSuite) TestExplodedDNS() {
+
+	var countResult struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// test exploded_dns and pdns values for some fqdns
+	tests := []struct {
+		name        string
+		fqdn        string
+		subdomains  uint64
+		visits      uint64
+		resolvedIPs uint64
+	}{
+		{
+			name:        "Root Domain",
+			fqdn:        "microsoft.com",
+			subdomains:  67,
+			visits:      1687,
+			resolvedIPs: 29,
+		},
+		{
+			name:        "Subdomain",
+			fqdn:        "dnsc.r-1x.com",
+			subdomains:  62466,
+			visits:      108911,
+			resolvedIPs: 0,
+		},
+		{
+			name:        "C2 Over DNS",
+			fqdn:        "r-1x.com",
+			subdomains:  62468,
+			visits:      109227,
+			resolvedIPs: 1,
+		},
+	}
+
+	for _, test := range tests {
+		it.Run(test.name, func() {
+			t := it.T()
+
+			ctx := clickhouse.Context(it.db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+				"fqdn": test.fqdn,
+			}))
+
+			var result struct {
+				Subdomains uint64 `ch:"subdomains"`
+				Visits     uint64 `ch:"visits"`
+			}
+
+			err := it.db.Conn.QueryRow(ctx, `
+				SELECT uniqExactMerge(subdomains) as subdomains, countMerge(visits) as visits FROM exploded_dns
+				WHERE fqdn = {fqdn:String}
+			`).ScanStruct(&result)
+			require.NoError(t, err, "querying exploded_dns table should not produce an error")
+			require.Equal(t, test.subdomains, result.Subdomains, "exploded_dns subdomain count should match expected value")
+			require.Equal(t, test.visits, result.Visits, "exploded_dns visit count should match expected value")
+
+			err = it.db.Conn.QueryRow(ctx, `
+				SELECT countIf(endsWith(query, concat('.', {fqdn:String})) OR query = {fqdn:String}) as count FROM dns
+			`).ScanStruct(&countResult)
+			require.NoError(t, err, "querying dns table should not produce an error")
+			require.EqualValues(t, test.visits, countResult.Count, "dns visit count should match expected value")
+
+			err = it.db.Conn.QueryRow(ctx, `
+				SELECT count(DISTINCT resolved_ip) as count FROM pdns
+				WHERE endsWith(fqdn, {fqdn:String})
+			`).ScanStruct(&countResult)
+			require.NoError(t, err, "querying pdns table should not produce an error")
+			require.EqualValues(t, test.resolvedIPs, countResult.Count, "pdns resolved_ips count should match expected value")
+
+		})
+	}
+
+	// ensure that there are no fqdns that are empty strings, only dots, or don't contain a dot
+	it.Run("No FQDNS Empty", func() {
+		t := it.T()
+		err := it.db.Conn.QueryRow(it.db.GetContext(), `
+			SELECT count() as count FROM exploded_dns
+			WHERE fqdn = '' OR fqdn = ' ' OR fqdn = '.' OR position('.' IN fqdn) == 0
+		`).ScanStruct(&countResult)
+		require.NoError(t, err, "querying exploded_dns table should not produce an error")
+		require.EqualValues(t, 0, countResult.Count, "fqdn fields in exploded_dns table must not be malformed")
+	})
+
+	// ensure that PDNS has the correct counts
+	it.Run("PDNS_Raw Counts", func() {
+		t := it.T()
+		err := it.db.Conn.QueryRow(it.db.GetContext(), `
+			SELECT count() as count FROM pdns_raw
+		`).ScanStruct(&countResult)
+		require.NoError(t, err, "querying pdns_raw table should not produce an error")
+		require.EqualValues(t, 208296, countResult.Count, "pdns_raw count should match expected value")
+	})
+
+}
+
+// testUConn tests the uconn table fields, values and hourly counts
+func (it *ValidDatasetTestSuite) TestUconn() {
+
+	type hourlyCount struct {
+		Count uint64 `ch:"count"`
+		Hour  uint32 `ch:"hour_timestamp"`
+	}
+
+	// test counts per hour for a unique connection pair
+	tests := []struct {
+		src               string
+		dst               string
+		localSrc          bool
+		localDst          bool
+		count             int64
+		totalSrcBytes     int64
+		totalDstBytes     int64
+		totalSrcIPBytes   int
+		totalDstIPBytes   int
+		totalIPBytes      int
+		totalSrcPackets   int
+		totalDstPackets   int
+		totalDuration     float64
+		tsListLen         int
+		srcIPBytesListLen int
+		firstSeen         int
+		lastSeen          int
+		hourlyCounts      []hourlyCount
+	}{
+		{
+			src: "192.168.88.2", dst: "165.227.88.15",
+			localSrc:          true,
+			localDst:          false,
+			count:             108858,
+			totalSrcBytes:     6723739,
+			totalDstBytes:     8900291,
+			totalSrcIPBytes:   9780272,
+			totalDstIPBytes:   11945399,
+			totalIPBytes:      21725671,
+			totalSrcPackets:   108870,
+			totalDstPackets:   108753,
+			totalDuration:     7588.427,
+			firstSeen:         1517336042, // 1517336042.279652
+			lastSeen:          1517422440, // 1517422440.290417
+			tsListLen:         86400,      // we max out tslist at 86400, actual number of unique ts is 108856
+			srcIPBytesListLen: 86400,      // we max out tslist at 86400, actual number of unique ts is 108858
+			hourlyCounts: []hourlyCount{
+				{Count: 1568, Hour: 1517335200},
+				{Count: 6255, Hour: 1517338800},
+				{Count: 5783, Hour: 1517342400},
+				{Count: 5126, Hour: 1517346000},
+				{Count: 4735, Hour: 1517349600},
+				{Count: 4512, Hour: 1517353200},
+				{Count: 4407, Hour: 1517356800},
+				{Count: 4415, Hour: 1517360400},
+				{Count: 4399, Hour: 1517364000},
+				{Count: 4470, Hour: 1517367600},
+				{Count: 4481, Hour: 1517371200},
+				{Count: 4464, Hour: 1517374800},
+				{Count: 4438, Hour: 1517378400},
+				{Count: 4452, Hour: 1517382000},
+				{Count: 4481, Hour: 1517385600},
+				{Count: 4377, Hour: 1517389200},
+				{Count: 4510, Hour: 1517392800},
+				{Count: 4451, Hour: 1517396400},
+				{Count: 4394, Hour: 1517400000},
+				{Count: 4415, Hour: 1517403600},
+				{Count: 4346, Hour: 1517407200},
+				{Count: 4412, Hour: 1517410800},
+				{Count: 4432, Hour: 1517414400},
+				{Count: 4494, Hour: 1517418000},
+				{Count: 1041, Hour: 1517421600},
+			},
+		},
+		{
+			src: "10.55.100.111", dst: "165.227.216.194",
+			localSrc:          true,
+			localDst:          false,
+			count:             20054,
+			totalSrcBytes:     0,
+			totalDstBytes:     0,
+			totalSrcIPBytes:   1042860,
+			totalDstIPBytes:   802160,
+			totalIPBytes:      1845020,
+			totalSrcPackets:   20055,
+			totalDstPackets:   20054,
+			totalDuration:     1292.3005,
+			firstSeen:         1517336052, // 1517336052.713711
+			lastSeen:          1517422432, // 1517422432.999706
+			tsListLen:         20054,
+			srcIPBytesListLen: 20054,
+			hourlyCounts: []hourlyCount{
+				{Count: 642, Hour: 1517335200},
+				{Count: 837, Hour: 1517338800},
+				{Count: 836, Hour: 1517342400},
+				{Count: 831, Hour: 1517346000},
+				{Count: 825, Hour: 1517349600},
+				{Count: 837, Hour: 1517353200},
+				{Count: 852, Hour: 1517356800},
+				{Count: 828, Hour: 1517360400},
+				{Count: 831, Hour: 1517364000},
+				{Count: 834, Hour: 1517367600},
+				{Count: 831, Hour: 1517371200},
+				{Count: 843, Hour: 1517374800},
+				{Count: 843, Hour: 1517378400},
+				{Count: 837, Hour: 1517382000},
+				{Count: 837, Hour: 1517385600},
+				{Count: 828, Hour: 1517389200},
+				{Count: 840, Hour: 1517392800},
+				{Count: 831, Hour: 1517396400},
+				{Count: 834, Hour: 1517400000},
+				{Count: 838, Hour: 1517403600},
+				{Count: 833, Hour: 1517407200},
+				{Count: 828, Hour: 1517410800},
+				{Count: 843, Hour: 1517414400},
+				{Count: 834, Hour: 1517418000},
+				{Count: 201, Hour: 1517421600},
+			},
+		},
+		{
+			src: "10.55.200.10", dst: "216.239.34.10",
+			localSrc:          true,
+			localDst:          false,
+			count:             3856,
+			totalSrcBytes:     181438,
+			totalDstBytes:     280134,
+			totalSrcIPBytes:   289630,
+			totalDstIPBytes:   388326,
+			totalIPBytes:      677956,
+			totalSrcPackets:   3864,
+			totalDstPackets:   3864,
+			totalDuration:     337.444094,
+			firstSeen:         1517336164, // 1517336164.364496
+			lastSeen:          1517414232, // 1517414232.320150
+			tsListLen:         3856,
+			srcIPBytesListLen: 3856,
+			hourlyCounts: []hourlyCount{
+				{Count: 120, Hour: 1517335200},
+				{Count: 187, Hour: 1517338800},
+				{Count: 174, Hour: 1517342400},
+				{Count: 164, Hour: 1517346000},
+				{Count: 175, Hour: 1517349600},
+				{Count: 177, Hour: 1517353200},
+				{Count: 174, Hour: 1517356800},
+				{Count: 187, Hour: 1517360400},
+				{Count: 186, Hour: 1517364000},
+				{Count: 164, Hour: 1517367600},
+				{Count: 164, Hour: 1517371200},
+				{Count: 166, Hour: 1517374800},
+				{Count: 201, Hour: 1517378400},
+				{Count: 197, Hour: 1517382000},
+				{Count: 179, Hour: 1517385600},
+				{Count: 188, Hour: 1517389200},
+				{Count: 174, Hour: 1517392800},
+				{Count: 195, Hour: 1517396400},
+				{Count: 169, Hour: 1517400000},
+				{Count: 178, Hour: 1517403600},
+				{Count: 170, Hour: 1517407200},
+				{Count: 167, Hour: 1517410800},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		it.Run(test.src+"-"+test.dst, func() {
+			t := it.T()
+			ctx := clickhouse.Context(it.db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+				"src": test.src,
+				"dst": test.dst,
+			}))
+
+			t.Run("Values", func(t *testing.T) {
+				type res struct {
+					importer.ConnEntry
+					LocalSrc     bool   `ch:"src_local"`
+					LocalDst     bool   `ch:"dst_local"`
+					TotalIPBytes int64  `ch:"total_ip_bytes"`
+					TSListLen    uint64 `ch:"ts_list_len"`
+					BytesListLen uint64 `ch:"bytes_list_len"`
+					Count        uint64 `ch:"count"`
+					FirstSeen    uint32 `ch:"first_seen"`
+					LastSeen     uint32 `ch:"last_seen"`
+				}
+				var result res
+				// conn count
+				err := it.db.Conn.QueryRow(ctx, `
+				SELECT src, dst, src_local, dst_local,
+					   countMerge(count) as count,
+					   sumMerge(total_src_bytes) as src_bytes,
+					   sumMerge(total_dst_bytes) as dst_bytes,
+					   sumMerge(total_src_ip_bytes) as src_ip_bytes,
+					   sumMerge(total_dst_ip_bytes) as dst_ip_bytes,
+					   sumMerge(total_ip_bytes) as total_ip_bytes,
+					   sumMerge(total_src_packets) as src_packets,
+					   sumMerge(total_dst_packets) as dst_packets,
+					   sumMerge(total_duration) as duration,
+					   length(groupArrayMerge(86400)(ts_list)) as ts_list_len,
+					   length(groupArrayMerge(86400)(src_ip_bytes_list)) as bytes_list_len,
+					   toUnixTimestamp(minMerge(first_seen)) as first_seen,
+					   toUnixTimestamp(maxMerge(last_seen)) as last_seen
+					   FROM uconn
+				WHERE src=={src:String} AND dst=={dst:String} 
+				GROUP BY src, dst, src_local, dst_local
+			`).ScanStruct(&result)
+				require.NoError(t, err, "querying uconn table should not produce an error")
+
+				require.EqualValues(t, test.count, result.Count, "count should match expected value")
+				require.EqualValues(t, test.localSrc, result.LocalSrc, "src_local should match expected value")
+				require.EqualValues(t, test.localDst, result.LocalDst, "dst_local should match expected value")
+				require.EqualValues(t, test.totalSrcBytes, result.SrcBytes, "total_src_bytes should match expected value")
+				require.EqualValues(t, test.totalDstBytes, result.DstBytes, "total_dst_bytes should match expected value")
+				require.EqualValues(t, test.totalSrcIPBytes, result.SrcIPBytes, "total_src_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalDstIPBytes, result.DstIPBytes, "total_dst_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalIPBytes, result.TotalIPBytes, "total_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalSrcPackets, result.SrcPackets, "total_src_packets should match expected value")
+				require.EqualValues(t, test.totalDstPackets, result.DstPackets, "total_dst_packets should match expected value")
+				require.InDelta(t, test.totalDuration, result.Duration, 0.01, "total_duration should match expected value")
+				require.EqualValues(t, test.tsListLen, result.TSListLen, "length of ts_list should match expected value")
+				require.EqualValues(t, test.srcIPBytesListLen, result.BytesListLen, "length of src_ip_bytes_list should match expected value")
+				require.EqualValues(t, test.firstSeen, result.FirstSeen, "first_seen should match expected value")
+				require.EqualValues(t, test.lastSeen, result.LastSeen, "last_seen should match expected value")
+			})
+
+			t.Run("Hourly Counts", func(t *testing.T) {
+				var res []hourlyCount
+
+				// select each hour for this connection pair and count its connections (per hour)
+				err := it.db.Conn.Select(ctx, &res, `
+					SELECT toUnixTimestamp(hour) AS hour_timestamp, countMerge(count) AS count FROM uconn
+					WHERE src=={src:String} AND dst=={dst:String}
+					GROUP BY hour
+					ORDER BY hour_timestamp
+				`)
+				require.NoError(t, err, "querying uconn should not produce an error")
+
+				// ensure that the number of hourly counts matches the expected number
+				require.EqualValues(t, len(test.hourlyCounts), len(res), "number of hourly count records must match expected value")
+
+				// ensure that hourly counts match expected values
+				require.Equal(t, test.hourlyCounts, res, "hourly counts must match expected values")
+			})
+		})
+	}
+
+}
+
+// testUSNI tests the usni table fields, values and hourly counts
+func (it *ValidDatasetTestSuite) TestUSNI() {
+	type hourlyCount struct {
+		Count uint64 `ch:"count"`
+		Hour  uint32 `ch:"hour_timestamp"`
+	}
+
+	type test struct {
+		src               string
+		fqdn              string
+		localSrc          bool
+		localDst          bool
+		count             int
+		httpCount         int
+		sslCount          int
+		totalSrcBytes     int
+		totalDstBytes     int
+		totalSrcIPBytes   int
+		totalDstIPBytes   int
+		totalIPBytes      int
+		totalSrcPackets   int
+		totalDstPackets   int
+		totalDuration     float64
+		firstSeen         int
+		lastSeen          int
+		tsListLen         int
+		uniqueTSListLen   int
+		srcIPBytesListLen int
+		serverIPs         []string
+		proxyIPs          []string
+		proxyCount        int
+		hourlyCounts      []hourlyCount
+	}
+
+	testCases := []test{
+		{
+			src: "10.55.100.109", fqdn: "www.alexa.com",
+			localSrc:          true,
+			localDst:          false,
+			count:             607,
+			httpCount:         290,
+			sslCount:          317,
+			totalSrcIPBytes:   700378,      // 127890 + 572488 = 700378
+			totalDstIPBytes:   23374587,    // 180570 + 23194017 = 23374587
+			totalIPBytes:      24074965,    // 308460 + 23766505 = 24074965
+			totalSrcBytes:     375130,      // 54810 + 320320 = 375130
+			totalDstBytes:     23018743,    // 116290 + 22902453 = 23018743
+			totalSrcPackets:   7944,        // 1740 + 6204 = 7944
+			totalDstPackets:   8714,        // 1520 + 7194 = 8714
+			totalDuration:     59949.50388, // 29042.726421 + 30906.777459 = 59949.50388
+			tsListLen:         607,         // 290 + 317 = 607
+			uniqueTSListLen:   354,
+			srcIPBytesListLen: 607,
+			firstSeen:         1517336154, // http: 1517336154.078555 ssl: 1517336154.222757
+			lastSeen:          1517422323, // http: 1517422323.066450 ssl: 1517422323.202331
+			serverIPs:         []string{"52.55.1.124", "34.196.128.45", "52.44.164.170", "34.198.172.204"},
+			proxyIPs:          []string{},
+			proxyCount:        0,
+			hourlyCounts: []hourlyCount{
+				{Count: 18, Hour: 1517335200},
+				{Count: 24, Hour: 1517338800},
+				{Count: 27, Hour: 1517342400},
+				{Count: 28, Hour: 1517346000},
+				{Count: 24, Hour: 1517349600},
+				{Count: 26, Hour: 1517353200},
+				{Count: 26, Hour: 1517356800},
+				{Count: 24, Hour: 1517360400},
+				{Count: 26, Hour: 1517364000},
+				{Count: 26, Hour: 1517367600},
+				{Count: 24, Hour: 1517371200},
+				{Count: 24, Hour: 1517374800},
+				{Count: 24, Hour: 1517378400},
+				{Count: 24, Hour: 1517382000},
+				{Count: 28, Hour: 1517385600},
+				{Count: 24, Hour: 1517389200},
+				{Count: 26, Hour: 1517392800},
+				{Count: 24, Hour: 1517396400},
+				{Count: 26, Hour: 1517400000},
+				{Count: 24, Hour: 1517403600},
+				{Count: 24, Hour: 1517407200},
+				{Count: 28, Hour: 1517410800},
+				{Count: 24, Hour: 1517414400},
+				{Count: 28, Hour: 1517418000},
+				{Count: 6, Hour: 1517421600},
+			},
+		},
+		{
+			src: "10.55.100.108", fqdn: "pixel.adsafeprotected.com",
+			localSrc:          true,
+			localDst:          false,
+			count:             163, // 132 + 31 = 163
+			httpCount:         132,
+			sslCount:          31,
+			totalSrcBytes:     105463,     // 69233 + 36230 = 105463
+			totalDstBytes:     626545,     // 129423 + 497122 = 626545
+			totalSrcIPBytes:   151619,     // 98537 + 53082 = 151619
+			totalDstIPBytes:   669573,     // 158087 + 511486 = 669573
+			totalIPBytes:      821192,     // 256624 + 564568 = 821192
+			totalSrcPackets:   1105,       // 693 + 412 = 1105
+			totalDstPackets:   1027,       // 677 + 350 = 1027
+			totalDuration:     917.166567, // 390.535353 + 526.631214 = 917.166567
+			tsListLen:         163,        // 132 + 31 = 163
+			uniqueTSListLen:   78,
+			srcIPBytesListLen: 163,
+			firstSeen:         1517344325, // http: 1517344325.773730 ssl: 1517344330.582307
+			lastSeen:          1517420230, // http: 1517420229.724142 ssl: 1517420230.547946
+			serverIPs:         []string{"69.172.216.55"},
+			proxyIPs:          []string{},
+			proxyCount:        0,
+			hourlyCounts: []hourlyCount{
+				{Count: 20, Hour: 1517342400},
+				{Count: 10, Hour: 1517346000},
+				{Count: 8, Hour: 1517349600},
+				{Count: 15, Hour: 1517356800},
+				{Count: 11, Hour: 1517364000},
+				{Count: 10, Hour: 1517371200},
+				{Count: 10, Hour: 1517374800},
+				{Count: 13, Hour: 1517378400},
+				{Count: 3, Hour: 1517382000},
+				{Count: 16, Hour: 1517385600},
+				{Count: 10, Hour: 1517389200},
+				{Count: 1, Hour: 1517396400},
+				{Count: 10, Hour: 1517400000},
+				{Count: 8, Hour: 1517410800},
+				{Count: 8, Hour: 1517414400},
+				{Count: 10, Hour: 1517418000},
+			},
+		},
+		{
+			src: "10.55.100.105", fqdn: "tile-service.weather.microsoft.com",
+			localSrc:          true,
+			localDst:          false,
+			count:             48,
+			httpCount:         48,
+			sslCount:          0,
+			totalSrcBytes:     10224,
+			totalDstBytes:     221820,
+			totalSrcIPBytes:   24012,
+			totalDstIPBytes:   234248,
+			totalIPBytes:      258260,
+			totalSrcPackets:   324,
+			totalDstPackets:   290,
+			totalDuration:     1533.756826, // 1533.756826
+			tsListLen:         48,
+			uniqueTSListLen:   48,
+			srcIPBytesListLen: 48,
+			firstSeen:         1517336820, // http: 1517336820.525381 ssl: -
+			lastSeen:          1517421421, // http: 1517421421.571744 ssl: -
+			serverIPs:         []string{"23.52.161.212", "23.63.158.27", "23.4.4.31", "23.222.23.103", "23.63.179.115", "23.79.207.65"},
+			proxyIPs:          []string{},
+			proxyCount:        0,
+			hourlyCounts: []hourlyCount{
+				{Count: 2, Hour: 1517335200},
+				{Count: 2, Hour: 1517338800},
+				{Count: 2, Hour: 1517342400},
+				{Count: 2, Hour: 1517346000},
+				{Count: 2, Hour: 1517349600},
+				{Count: 2, Hour: 1517353200},
+				{Count: 2, Hour: 1517356800},
+				{Count: 2, Hour: 1517360400},
+				{Count: 2, Hour: 1517364000},
+				{Count: 2, Hour: 1517367600},
+				{Count: 2, Hour: 1517371200},
+				{Count: 2, Hour: 1517374800},
+				{Count: 2, Hour: 1517378400},
+				{Count: 2, Hour: 1517382000},
+				{Count: 2, Hour: 1517385600},
+				{Count: 2, Hour: 1517389200},
+				{Count: 2, Hour: 1517392800},
+				{Count: 2, Hour: 1517396400},
+				{Count: 2, Hour: 1517400000},
+				{Count: 2, Hour: 1517403600},
+				{Count: 2, Hour: 1517407200},
+				{Count: 2, Hour: 1517410800},
+				{Count: 2, Hour: 1517414400},
+				{Count: 2, Hour: 1517418000},
+			},
+		},
+		{
+			src:               "10.55.100.104",
+			fqdn:              "www.facebook.com",
+			localSrc:          true,
+			localDst:          false,
+			count:             183,
+			httpCount:         0,
+			sslCount:          183,
+			totalSrcIPBytes:   259611,
+			totalDstIPBytes:   836534,
+			totalIPBytes:      1096145,
+			totalSrcBytes:     152215,
+			totalDstBytes:     726418,
+			totalSrcPackets:   2630,
+			totalDstPackets:   2698,
+			totalDuration:     12005.465565,
+			tsListLen:         183,
+			uniqueTSListLen:   119,
+			srcIPBytesListLen: 183,
+			firstSeen:         1517336527, // 1517336527.546436
+			lastSeen:          1517422327, // 1517422327.673738
+			serverIPs:         []string{"157.240.2.35"},
+			proxyIPs:          []string{},
+			proxyCount:        0,
+			hourlyCounts: []hourlyCount{
+				{Count: 5, Hour: 1517335200},
+				{Count: 12, Hour: 1517338800},
+				{Count: 9, Hour: 1517342400},
+				{Count: 4, Hour: 1517346000},
+				{Count: 10, Hour: 1517349600},
+				{Count: 11, Hour: 1517353200},
+				{Count: 16, Hour: 1517356800},
+				{Count: 3, Hour: 1517360400},
+				{Count: 9, Hour: 1517364000},
+				{Count: 3, Hour: 1517367600},
+				{Count: 5, Hour: 1517371200},
+				{Count: 10, Hour: 1517374800},
+				{Count: 8, Hour: 1517378400},
+				{Count: 7, Hour: 1517382000},
+				{Count: 6, Hour: 1517385600},
+				{Count: 5, Hour: 1517389200},
+				{Count: 9, Hour: 1517392800},
+				{Count: 9, Hour: 1517396400},
+				{Count: 5, Hour: 1517400000},
+				{Count: 9, Hour: 1517403600},
+				{Count: 4, Hour: 1517407200},
+				{Count: 9, Hour: 1517410800},
+				{Count: 8, Hour: 1517414400},
+				{Count: 3, Hour: 1517418000},
+				{Count: 4, Hour: 1517421600},
+			},
+		},
+	}
+
+	var result struct {
+		Count             uint64   `ch:"count"`
+		LocalSrc          bool     `ch:"src_local"`
+		LocalDst          bool     `ch:"dst_local"`
+		TotalSrcIPBytes   int64    `ch:"total_src_ip_bytes"`
+		TotalDstIPBytes   int64    `ch:"total_dst_ip_bytes"`
+		TotalIPBytes      int64    `ch:"total_ip_bytes"`
+		TotalSrcBytes     int64    `ch:"total_src_bytes"`
+		TotalDstBytes     int64    `ch:"total_dst_bytes"`
+		TotalSrcPackets   int64    `ch:"total_src_packets"`
+		TotalDstPackets   int64    `ch:"total_dst_packets"`
+		TotalDuration     float64  `ch:"total_duration"`
+		TSListLen         uint64   `ch:"ts_list_len"`
+		UniqueTSListLen   uint64   `ch:"unique_ts_list_len"`
+		UniqueTSCount     uint64   `ch:"unique_ts_count"`
+		SrcIPBytesListLen uint64   `ch:"src_ip_bytes_list_len"`
+		FirstSeen         uint32   `ch:"first_seen"`
+		LastSeen          uint32   `ch:"last_seen"`
+		ServerIPs         []string `ch:"server_ips"`
+		ProxyIPs          []string `ch:"proxy_ips"`
+		ProxyCount        uint64   `ch:"proxy_count"`
+		ProxyBoolCount    uint64   `ch:"proxy_bool_count"`
+	}
+
+	for _, test := range testCases {
+		it.Run(test.src+"-"+test.fqdn, func() {
+			t := it.T()
+
+			ctx := clickhouse.Context(it.db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+				"src":  test.src,
+				"fqdn": test.fqdn,
+			}))
+
+			t.Run("Values", func(t *testing.T) {
+				err := it.db.Conn.QueryRow(ctx, `
+			SELECT 
+				src_local, dst_local,
+				countMerge(count) as count,
+				sumMerge(total_src_ip_bytes) as total_src_ip_bytes,
+				sumMerge(total_dst_ip_bytes) as total_dst_ip_bytes,
+				sumMerge(total_ip_bytes) as total_ip_bytes,
+				sumMerge(total_src_bytes) as total_src_bytes,
+				sumMerge(total_dst_bytes) as total_dst_bytes,
+				sumMerge(total_src_packets) as total_src_packets,
+				sumMerge(total_dst_packets) as total_dst_packets,
+				sumMerge(total_duration) as total_duration,
+				length(groupArrayMerge(86400)(ts_list)) as ts_list_len,
+				length(arrayDistinct(groupArrayMerge(86400)(ts_list))) as unique_ts_list_len,
+				uniqExactMerge(unique_ts_count) as unique_ts_count,
+				length(groupArrayMerge(86400)(src_ip_bytes_list)) as src_ip_bytes_list_len,
+				toUnixTimestamp(minMerge(first_seen)) as first_seen,
+				toUnixTimestamp(maxMerge(last_seen)) as last_seen,
+				groupUniqArrayMerge(10)(server_ips) as server_ips,
+				groupUniqArrayMerge(10)(proxy_ips) as proxy_ips,
+				countMerge(proxy_count) as proxy_count,
+				countIf(proxy) as proxy_bool_count
+			FROM usni
+			WHERE src={src:String} AND fqdn={fqdn:String}
+			GROUP BY src, fqdn, src_local, dst_local
+		`).ScanStruct(&result)
+				require.NoError(t, err)
+
+				require.EqualValues(t, test.count, int(result.Count), "conn count should match expected value")
+				require.EqualValues(t, test.totalSrcIPBytes, result.TotalSrcIPBytes, "total_src_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalDstIPBytes, int(result.TotalDstIPBytes), "total_dst_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalIPBytes, result.TotalIPBytes, "total_ip_bytes should match expected value")
+				require.EqualValues(t, test.totalSrcBytes, result.TotalSrcBytes, "total_src_bytes should match expected value")
+				require.EqualValues(t, test.totalDstBytes, result.TotalDstBytes, "total_dst_bytes should match expected value")
+				require.EqualValues(t, test.totalSrcPackets, result.TotalSrcPackets, "total_src_packets should match expected value")
+				require.EqualValues(t, test.totalDstPackets, result.TotalDstPackets, "total_dst_packets should match expected value")
+				require.InDelta(t, test.totalDuration, result.TotalDuration, 0.1, "total_duration should match expected value")
+				require.EqualValues(t, test.tsListLen, int(result.TSListLen), "number of elements in ts_list field should match expected value")
+				require.EqualValues(t, test.uniqueTSListLen, int(result.UniqueTSListLen), "number of unique elements in ts_list field should match expected value")
+				require.EqualValues(t, test.uniqueTSListLen, int(result.UniqueTSCount), "unique_ts_count should match expected value")
+				require.EqualValues(t, test.srcIPBytesListLen, result.SrcIPBytesListLen, "number of elements in src_ip_bytes_list field should match expected value")
+				require.EqualValues(t, test.firstSeen, result.FirstSeen, "first_seen should match expected value")
+				require.EqualValues(t, test.lastSeen, result.LastSeen, "last_seen should match expected value")
+				require.ElementsMatch(t, test.serverIPs, result.ServerIPs, "server_ips should match expected value")
+				require.ElementsMatch(t, test.proxyIPs, result.ProxyIPs, "proxy_ips should match expected value")
+				require.EqualValues(t, test.proxyCount, result.ProxyCount, "proxy_count should match expected value")
+				require.EqualValues(t, test.proxyCount, result.ProxyBoolCount, "proxy field boolean count should match proxy_count")
+
+				// separate queries for httpCount and sslCount
+				var resCount uint64
+				err = it.db.Conn.QueryRow(ctx, `
+					SELECT countMerge(count) FROM usni 
+					WHERE src={src:String} AND fqdn={fqdn:String} AND http=true 
+					GROUP BY src, fqdn
+				`).Scan(&resCount)
+				if err != nil && !errors.Is(err, sql.ErrNoRows) {
+					require.NoError(t, err)
+				}
+				if errors.Is(err, sql.ErrNoRows) {
+					resCount = 0
+				}
+				require.EqualValues(t, test.httpCount, int(resCount), "http conn count should match expected value")
+
+				err = it.db.Conn.QueryRow(ctx, `
+					SELECT countMerge(count) as ssl_count FROM usni 
+					WHERE src={src:String} AND fqdn={fqdn:String} AND http=false 
+					GROUP BY src, fqdn
+				`).Scan(&resCount)
+				if err != nil && !errors.Is(err, sql.ErrNoRows) {
+					require.NoError(t, err)
+				}
+				if errors.Is(err, sql.ErrNoRows) {
+					resCount = 0
+				}
+				require.EqualValues(t, test.sslCount, int(resCount), "ssl conn count should match expected value")
+
+			})
+
+			t.Run("Hourly Counts", func(t *testing.T) {
+				var res []hourlyCount
+
+				// select each hour for this connection pair and count its connections (per hour)
+				err := it.db.Conn.Select(ctx, &res, `
+					SELECT toUnixTimestamp(hour) AS hour_timestamp, countMerge(count) AS count FROM usni
+					WHERE src=={src:String} AND fqdn=={fqdn:String}
+					GROUP BY hour
+					ORDER BY hour_timestamp
+				`)
+				require.NoError(t, err, "querying usni should not produce an error")
+
+				// ensure that the number of hourly counts matches the expected number
+				require.EqualValues(t, len(test.hourlyCounts), len(res), "number of hourly count records must match expected value")
+
+				// ensure that hourly counts match expected values
+				require.Equal(t, test.hourlyCounts, res, "hourly counts must match expected values")
+			})
+		})
+	}
+}
+
+func (it *ValidDatasetTestSuite) TestLongConnections() {
+	t := it.T()
+
+	type testCase struct {
+		src           string
+		dst           string
+		totalDuration float64
+	}
+
+	topThree := []testCase{
+		{
+			src:           "::ffff:10.55.100.100",
+			dst:           "::ffff:65.52.108.225",
+			totalDuration: 86222.3655,
+		},
+		{
+			src:           "::ffff:10.55.100.107",
+			dst:           "::ffff:111.221.29.113",
+			totalDuration: 86220.1262,
+		},
+		{
+			src:           "::ffff:10.55.100.110",
+			dst:           "::ffff:40.77.229.82",
+			totalDuration: 86160.1197,
+		},
+	}
+	// bottom 3 (total duration) over 5hr threshold
+	bottomThree := []testCase{
+		{
+			src:           "::ffff:10.55.100.111",
+			dst:           "::ffff:34.196.128.45",
+			totalDuration: 18015.6872,
+		},
+		{
+			src:           "::ffff:10.55.100.106",
+			dst:           "::ffff:34.196.128.45",
+			totalDuration: 18055.099200000008,
+		},
+		{
+			src:           "::ffff:10.55.100.104",
+			dst:           "::ffff:172.217.8.196",
+			totalDuration: 18145.25299999999,
+		},
+	}
+	type result struct {
+		Src      string  `ch:"src"`
+		Dst      string  `ch:"dst"`
+		TotalDur float64 `ch:"total_duration"`
+	}
+
+	t.Run("Top 3 Long Connections", func(t *testing.T) {
+		var res []result
+		err := it.db.Conn.Select(it.db.GetContext(), &res, `
+			SELECT IPv6NumToString(src) as src, IPv6NumToString(dst) as dst, sumMerge(total_duration) as total_duration FROM uconn
+			GROUP BY src, dst
+			ORDER BY total_duration DESC LIMIT 3
+		`)
+		require.NoError(t, err)
+		require.Len(t, res, len(topThree), "length of result list should match expected value")
+		for i, r := range res {
+			require.Equal(t, topThree[i].src, r.Src, "uconn: total duration should match (top #%d src: %s, fqdn: %s)", i, topThree[i].src, topThree[i].dst)
+			require.Equal(t, topThree[i].dst, r.Dst, "uconn: total duration should match (bottom #%d src: %s, fqdn: %s)", i, topThree[i].src, topThree[i].dst)
+			require.InEpsilon(t, topThree[i].totalDuration, r.TotalDur, 0.3, "uconn: total duration should match (top #1%d src: %s, fqdn: %s, duration: %f)", i, topThree[i].src, topThree[i].dst, topThree[i].totalDuration)
+		}
+	})
+
+	t.Run("Bottom 3 Long Connections >5hrs", func(t *testing.T) {
+		var res []result
+		err := it.db.Conn.Select(it.db.GetContext(), &res, `
+			SELECT * FROM (
+				SELECT IPv6NumToString(src) as src, IPv6NumToString(dst) as dst, sumMerge(total_duration) as total_duration FROM uconn
+				GROUP BY src, dst
+			)
+			WHERE total_duration > 18000
+			ORDER BY total_duration ASC LIMIT 3
+		`)
+		require.NoError(t, err)
+		require.Len(t, res, len(bottomThree), "length of result list should match expected value")
+		for i, r := range res {
+			require.Equal(t, bottomThree[i].src, r.Src, "uconn: total duration should match (bottom #%d src: %s, fqdn: %s)", i, bottomThree[i].src, bottomThree[i].dst)
+			require.Equal(t, bottomThree[i].dst, r.Dst, "uconn: total duration should match (bottom #%d src: %s, fqdn: %s)", i, bottomThree[i].src, bottomThree[i].dst)
+			require.InEpsilon(t, bottomThree[i].totalDuration, r.TotalDur, 0.3, "uconn: total duration should match (bottom #1%d src: %s, fqdn: %s)", i, bottomThree[i].src, bottomThree[i].dst)
+		}
+	})
+}

integration/metadb_test.go

@@ -0,0 +1,498 @@
+package integration_test
+
+import (
+	"activecm/rita/cmd"
+	"activecm/rita/config"
+	"activecm/rita/database"
+	i "activecm/rita/importer"
+	"activecm/rita/util"
+	"context"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	fp "path/filepath"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func CheckImportFileTracking(t *testing.T, importer *i.Importer) { // uses valid dataset
+	t.Helper()
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// set context with importID and database parameters
+	ctx := clickhouse.Context(importer.Database.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"importID": importer.ImportID.Hex(),
+		"database": importer.Database.GetSelectedDB(),
+	}))
+
+	// make sure total (post duplicate filtering) file count to be imported matches file count in metadb
+	err := importer.Database.Conn.QueryRow(ctx, `
+			SELECT count() AS count FROM metadatabase.files
+			WHERE import_id = unhex({importID:String}) AND database = {database:String}
+		`).ScanStruct(&result)
+	require.NoError(t, err)
+	require.EqualValues(t, importer.TotalFileCount, result.Count, "total file count matches imported file count")
+
+	var allFiles []string
+	allFiles = append(allFiles, importer.FileMap[i.ConnPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.OpenConnPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.HTTPPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.OpenHTTPPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.SSLPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.OpenSSLPrefix]...)
+	allFiles = append(allFiles, importer.FileMap[i.DNSPrefix]...)
+
+	var filesResult struct {
+		Files []string `ch:"files"`
+	}
+
+	// set context with importID parameter
+	ctx = clickhouse.Context(importer.Database.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"importID": importer.ImportID.Hex(),
+	}))
+
+	// make sure all files in created filemap were imported and saved in metadb
+	err = importer.Database.Conn.QueryRow(ctx, `
+			SELECT groupArray(path) as files FROM metadatabase.files
+			WHERE import_id = unhex({importID:String})
+		`).ScanStruct(&filesResult)
+	require.NoError(t, err)
+	require.ElementsMatch(t, allFiles, filesResult.Files, "files in filemap created for import match list of imported files in metadb")
+
+	// Make sure duplicate files in same import don't get imported (conn.log & conn.log.gz)
+	for _, file := range allFiles {
+		if strings.HasSuffix(file, ".gz") {
+			hasUncompressedDuplicate := slices.Contains(allFiles, strings.TrimSuffix(file, ".gz"))
+			require.False(t, hasUncompressedDuplicate, "ucompressed duplicate of .gz file was not imported")
+		} else {
+			hasCompressedDuplicate := slices.Contains(allFiles, (file + ".gz"))
+			require.False(t, hasCompressedDuplicate, "compressed duplicate of uncompressed file was not imported")
+		}
+	}
+
+	// importing again should fail because all files are imported
+	err = importer.Import(afero.NewOsFs(), importer.FileMap)
+	require.Error(t, err)
+
+}
+
+func TestImportTracking(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// load config
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	// update config with clickhouse connection
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not produce an error")
+
+	// ROLLING IMPORT
+	// new import
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/open_conns/closed", "test_import_rolling", true, false)
+	require.NoError(t, err, "new rolling import should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_rolling", cfg, nil)
+	require.NoError(t, err)
+
+	// import another folder
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/open", "test_import_rolling", true, false)
+	require.NoError(t, err, "importing another folder to a rolling database should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_rolling", cfg, nil)
+	require.NoError(t, err)
+
+	// rebuild dataset
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/open", "test_import_rolling", true, true)
+	require.NoError(t, err, "importing same folder to a rebuilt rolling database should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_rolling", cfg, nil)
+	require.NoError(t, err)
+
+	// ***************************************************************************
+	// NON-ROLLING IMPORT
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/closed", "test_import_nonrolling", false, true)
+	require.NoError(t, err, "new non-rolling import should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_nonrolling", cfg, nil)
+	require.NoError(t, err)
+
+	// import another folder
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/closed", "test_import_nonrolling", false, false)
+	require.Error(t, err, "importing another folder to a non-rolling database should not succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_nonrolling", cfg, nil)
+	require.NoError(t, err)
+
+	// rebuild dataset
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/closed", "test_import_nonrolling", false, true)
+	require.NoError(t, err, "importing same folder to a rebuilt non-rolling database should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_nonrolling", cfg, nil)
+	require.NoError(t, err)
+
+	// rebuild dataset & convert to rolling
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/open", "test_import_nonrolling", true, true)
+	require.NoError(t, err, "importing once to a non-rolling database converted to a rolling database should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_nonrolling", cfg, nil)
+	require.NoError(t, err)
+
+	// import again to rolling dataset
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afero.NewOsFs(), TestDataPath+"/open_conns/closed", "test_import_nonrolling", true, true)
+	require.NoError(t, err, "importing twice to a converted rolling database should succeed")
+	// connect to database
+	_, err = database.ConnectToDB(context.Background(), "test_import_nonrolling", cfg, nil)
+	require.NoError(t, err)
+
+}
+
+// TestMinMaxTimestamps tests that the min and max timestamps are correctly stored in the metadatabase.imports table
+func TestMinMaxTimestamps(t *testing.T) {
+
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// load config
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	err = cfg.ResetConfig()
+	require.NoError(t, err)
+	cfg, err = config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+	// update config with clickhouse connection
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	require.True(t, cfg.Filter.FilterExternalToInternal)
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not produce an error")
+
+	// connect to clickhouse server
+	server, err := database.ConnectToServer(context.Background(), cfg)
+	require.NoError(t, err, "connecting to server should not produce an error")
+
+	err = server.Conn.Exec(server.GetContext(), "TRUNCATE TABLE IF EXISTS metadatabase.imports")
+	require.NoError(t, err)
+
+	err = server.Conn.Exec(server.GetContext(), "TRUNCATE TABLE IF EXISTS metadatabase.min_max")
+	require.NoError(t, err)
+
+	t.Run("Open Dataset", func(t *testing.T) {
+
+		// open dataset
+		results, err := cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/open_conns/open", "test_minmax_open", false, true)
+		require.NoError(t, err, "import should succeed")
+
+		// connect to database
+		db, err := database.ConnectToDB(context.Background(), "test_minmax_open", cfg, nil)
+		require.NoError(t, err)
+
+		minTSBeacon := 1517420070
+		maxTSBeacon := 1517422419
+		minTS := 1517336019
+		maxTS := 1517422419
+		minOpenTS := 1517336042
+		maxOpenTS := 1517336223
+
+		// validate metadb imports table values for min/max (these are used for troubleshooting)
+		min, max, minOpen, maxOpen := getMinMaxTimestamps(t, db, results.ImportID[0])
+		require.EqualValues(t, minTS, min.Unix(), "imports: min timestamp matches expected min timestamp: open_conns/open")
+		require.EqualValues(t, maxTS, max.Unix(), "imports: max timestamp matches expected max timestamp: open_conns/open")
+		require.EqualValues(t, minOpenTS, minOpen.Unix(), "imports: min open timestamp matches expected min open timestamp: open_conns/open")
+		require.EqualValues(t, maxOpenTS, maxOpen.Unix(), "imports: max open timestamp matches expected max open timestamp: open_conns/open")
+
+		// validate metadb min_max values for min/max
+		min, max, notFromConn, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "getting true min/max timestamps should not error")
+		require.EqualValues(t, minTS, min.Unix(), "min_max: min timestamp matches expected min timestamp: open_conns/open")
+		require.EqualValues(t, maxTS, max.Unix(), "min_max: max timestamp matches expected max timestamp: open_conns/open")
+
+		// validate which table min max is from
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+		require.False(t, useCurrentTime, "first seen analysis should not use the current time")
+
+		min, max, notFromConn, err = db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "getting beacon min/max timestamps should not error")
+		require.EqualValues(t, minTSBeacon, min.Unix(), "min_max: beacon min timestamp matches expected min timestamp: open_conns/open")
+		require.EqualValues(t, maxTSBeacon, max.Unix(), "min_max: beacon max timestamp matches expected max timestamp: open_conns/open")
+
+		// validate which table min max is from
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+	})
+
+	t.Run("Closed Dataset", func(t *testing.T) {
+
+		// closed dataset
+		results, err := cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/valid_tsv", "test_minmax", false, true)
+		require.NoError(t, err, "import should succeed")
+
+		// connect to database
+		db, err := database.ConnectToDB(context.Background(), "test_minmax", cfg, nil)
+		require.NoError(t, err)
+
+		min, max, minOpen, maxOpen := getMinMaxTimestamps(t, db, results.ImportID[0])
+		require.NoError(t, err, "getting min/max timestamps should not error")
+
+		minTS := 1517336040 // isn't 1517336042 because DNS rounds to start of hour
+		minOpenTS := 1517336042
+		maxTS := 1517422440
+		minTSBeacon := 1517336042
+		maxTSBeacon := 1517422440
+
+		// validate metadb imports table values for min/max (these are used for troubleshooting)
+		require.EqualValues(t, minTS, min.Unix(), "imports: min timestamp matches expected min timestamp: valid_tsv")
+		require.EqualValues(t, maxTS, max.Unix(), "imports: max timestamp matches expected max timestamp: valid_tsv")
+		require.EqualValues(t, minOpenTS, minOpen.Unix(), "imports: min open timestamp should be unset: valid_tsv")
+		require.EqualValues(t, maxTS, maxOpen.Unix(), "imports: max open timestamp should be unset: valid_tsv")
+
+		// validate metadb min_max values for min/max
+		min, max, notFromConn, useCurrentTime, err := db.GetTrueMinMaxTimestamps()
+		require.NoError(t, err, "getting min/max timestamps should not error")
+		require.EqualValues(t, minTS, min.Unix(), "min_max: min timestamp matches expected min timestamp: open_conns/open")
+		require.EqualValues(t, maxTS, max.Unix(), "min_max: max timestamp matches expected max timestamp: open_conns/open")
+
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+		require.False(t, useCurrentTime, "first seen analysis should not use the current time")
+
+		min, max, notFromConn, err = db.GetBeaconMinMaxTimestamps()
+		require.NoError(t, err, "getting min/max timestamps should not error")
+		require.EqualValues(t, minTSBeacon, min.Unix(), "min_max: beacon min timestamp matches expected min timestamp: open_conns/open")
+		require.EqualValues(t, maxTSBeacon, max.Unix(), "min_max: beacon max timestamp matches expected max timestamp: open_conns/open")
+
+		require.False(t, notFromConn, "min and max timestamps should be from conn table")
+
+	})
+}
+
+func getMinMaxTimestamps(t *testing.T, db *database.DB, importID util.FixedString) (time.Time, time.Time, time.Time, time.Time) { // uses valid dataset
+	t.Helper()
+
+	type minMaxRes struct {
+		Min     time.Time `ch:"min_timestamp"`
+		Max     time.Time `ch:"max_timestamp"`
+		MinOpen time.Time `ch:"min_open_timestamp"`
+		MaxOpen time.Time `ch:"max_open_timestamp"`
+	}
+
+	var result minMaxRes
+
+	// set context with importID and database parameters
+	ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"importID": importID.Hex(),
+		"database": db.GetSelectedDB(),
+	}))
+
+	// make sure min and max timestamps are correct in the metadatabase.imports record
+	err := db.Conn.QueryRow(ctx, `
+			SELECT min_timestamp, max_timestamp, min_open_timestamp, max_open_timestamp FROM metadatabase.imports
+			WHERE import_id = unhex({importID:String}) AND database = {database:String}
+			ORDER BY ended_at DESC
+		`).ScanStruct(&result)
+	require.NoError(t, err)
+	return result.Min, result.Max, result.MinOpen, result.MaxOpen
+}
+
+func TestMetaDatabase(t *testing.T) {
+	// set up file system interface
+	afs := afero.NewOsFs()
+
+	// load config
+	cfg, err := config.LoadConfig(afs, ConfigPath)
+	require.NoError(t, err)
+
+	cfg.DBConnection = dockerInfo.clickhouseConnection
+	err = config.UpdateConfig(cfg)
+	require.NoError(t, err, "updating config should not produce an error")
+
+	// import a dataset
+	dbName := "test_metadb"
+	results, err := cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/valid_tsv", dbName+"_tsv", false, true)
+	require.NoError(t, err, "import should succeed")
+	importID := results.ImportID[0]
+
+	// import a few other datasets for testing multiple metadb entries
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/valid_json", dbName+"_json", false, true)
+	require.NoError(t, err, "import should succeed")
+	_, err = cmd.RunImportCmd(time.Now(), cfg, afs, TestDataPath+"/open_conns/open", dbName+"_open", false, true)
+	require.NoError(t, err, "import should succeed")
+
+	// connect to metadatabase
+	db, err := database.ConnectToDB(context.Background(), "metadatabase", cfg, nil)
+	require.NoError(t, err)
+
+	// test metadatabase tables
+	t.Run("Imports Table", func(t *testing.T) {
+		validateMetaDBImportsTable(t, db, dbName+"_tsv", importID)
+	})
+	t.Run("Files Table", func(t *testing.T) {
+		validateMetaDBFilesTable(t, db, TestDataPath+"/valid_tsv", dbName+"_tsv", importID)
+	})
+	validateMetaDBHistoricalFirstSeenTable(t)
+	validateMetaDBValidMimeTypesTable(t)
+
+}
+func validateMetaDBImportsTable(t *testing.T, db *database.DB, dbName string, importID util.FixedString) {
+	t.Helper()
+
+	// set the context
+	ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"table":    "imports",
+		"database": dbName,
+		"importID": importID.Hex(),
+	}))
+
+	var result struct {
+		Count uint64 `ch:"count"`
+	}
+
+	// imports table exists
+	t.Run("Table Exists", func(t *testing.T) {
+		var exists uint8
+		err := db.Conn.QueryRow(ctx, `EXISTS TABLE {table:Identifier}`).Scan(&exists)
+		require.NoError(t, err)
+		require.EqualValues(t, 1, exists, "imports table exists")
+	})
+
+	// verify that start and end import records for first import were created
+	t.Run("Record Matches", func(t *testing.T) {
+		err := db.Conn.QueryRow(ctx, `
+			SELECT count() AS count 
+			FROM {table:Identifier}
+			WHERE database = {database:String} AND import_id = unhex({importID:String})
+			`).ScanStruct(&result)
+
+		require.NoError(t, err)
+		// 2 import records should be created, one at the start of the import and one when the import is finished
+		require.EqualValues(t, 2, result.Count, "expected 2 import records to be created, got %d", result.Count)
+	})
+
+	// no import record has unset import_id
+	t.Run("No Unset ImportID", func(t *testing.T) {
+		err := db.Conn.QueryRow(ctx, `
+			SELECT count() AS count 
+			FROM {table:Identifier}
+			WHERE import_id==toFixedString('',16) OR hex(import_id)=='00000000000000000000000000000000' OR import_id=='' OR import_id IS NULL
+			`).ScanStruct(&result)
+
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "imports table must not have unset import_id fields")
+	})
+
+	// no import record has unset database
+	t.Run("No Unset Database", func(t *testing.T) {
+		err := db.Conn.QueryRow(ctx, `
+			SELECT count() AS count 
+			FROM {table:Identifier}
+			WHERE database=='' OR database IS NULL
+			`).ScanStruct(&result)
+
+		require.NoError(t, err)
+		require.EqualValues(t, 0, result.Count, "imports table must not have unset database fields")
+	})
+
+}
+
+func validateMetaDBFilesTable(t *testing.T, db *database.DB, directory string, dbName string, importID util.FixedString) {
+	t.Helper()
+
+	// set the context
+	ctx := clickhouse.Context(db.GetContext(), clickhouse.WithParameters(clickhouse.Parameters{
+		"table":    "files",
+		"database": dbName,
+		"importID": importID.Hex(),
+	}))
+
+	// files table exists
+	t.Run("Table Exists", func(t *testing.T) {
+		var exists uint8
+		err := db.Conn.QueryRow(ctx, `EXISTS TABLE {table:Identifier}`).Scan(&exists)
+		require.NoError(t, err)
+		require.EqualValues(t, 1, exists, "files table must exist")
+	})
+
+	// verify that files entries for import are correct
+	t.Run("Files List Correct", func(t *testing.T) {
+
+		// get the files listed in the files table for the first import
+		rows, err := db.Conn.Query(ctx, `
+				SELECT path
+				FROM {table:Identifier}
+				WHERE database = {database:String} AND import_id = unhex({importID:String})
+			`)
+		require.NoError(t, err)
+		defer rows.Close()
+
+		// type fileVersions
+
+		// map to track the versions of each file
+		dbFiles := make(map[string]struct {
+			log   bool
+			logGz bool
+		})
+
+		for rows.Next() {
+			var result struct {
+				Path string `ch:"path"`
+			}
+			err = rows.ScanStruct(&result)
+			require.NoError(t, err)
+
+			// get the filename without the path
+			name := fp.Base(result.Path)
+
+			// set map key as the filename without the .gz extension
+			normalizedFilename := strings.TrimSuffix(name, ".gz")
+
+			// get the version info for the file
+			versionInfo := dbFiles[normalizedFilename]
+
+			// set the version info for the file
+			if strings.HasSuffix(name, ".gz") {
+				versionInfo.logGz = true
+			} else {
+				versionInfo.log = true
+			}
+
+			// update the map
+			dbFiles[normalizedFilename] = versionInfo
+		}
+
+		// get all files in the directory
+		fs := afero.NewOsFs()
+		allFiles, err := afero.ReadDir(fs, directory)
+		require.NoError(t, err)
+
+		// check that a single version (log or log.gz) of each file is present in the database
+		for _, file := range allFiles {
+			normalizedFileName := strings.TrimSuffix(file.Name(), ".gz")
+
+			if normalizedFileName == ".DS_Store" {
+				continue
+			}
+
+			versions, exists := dbFiles[normalizedFileName]
+
+			// check that a version of the file exists in the database
+			require.True(t, exists, "file must exist in database records: %s", normalizedFileName)
+
+			// verify that only one version of the file is present in the database
+			require.False(t, versions.log && versions.logGz, "both .log and .log.gz versions cannot be present in the database")
+		}
+	})
+}
+
+func validateMetaDBHistoricalFirstSeenTable(t *testing.T) {
+	t.Helper()
+}
+
+func validateMetaDBValidMimeTypesTable(t *testing.T) {
+	t.Helper()
+}