mirror of
https://github.com/projectdiscovery/httpx
synced 2026-06-08 16:50:17 +00:00
2ea576f175
Add support for storing httpx scan results directly to databases with both CLI flags and YAML config file options. Supported databases: - MongoDB - PostgreSQL - MySQL Features: - Batched writes for performance (configurable batch size) - Auto-flush with configurable interval - Individual columns for each Result field (not JSON blob) - Support for environment variable HTTPX_DB_CONNECTION_STRING - Option to omit raw request/response data (-rdbor) New CLI flags under OUTPUT group: - -rdb, -result-db: enable database storage - -rdbc, -result-db-config: path to YAML config file - -rdbt, -result-db-type: database type (mongodb, postgres, mysql) - -rdbcs, -result-db-conn: connection string - -rdbn, -result-db-name: database name (default: httpx) - -rdbtb, -result-db-table: table/collection name (default: results) - -rdbbs, -result-db-batch-size: batch size (default: 100) - -rdbor, -result-db-omit-raw: omit raw data Closes #1973 Closes #2360 Closes #2361 Closes #2362
71 lines
1.3 KiB
Go
71 lines
1.3 KiB
Go
package db
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/projectdiscovery/httpx/runner"
|
|
)
|
|
|
|
type DatabaseType string
|
|
|
|
const (
|
|
MongoDB DatabaseType = "mongodb"
|
|
PostgreSQL DatabaseType = "postgres"
|
|
MySQL DatabaseType = "mysql"
|
|
)
|
|
|
|
func (d DatabaseType) String() string {
|
|
return string(d)
|
|
}
|
|
|
|
func (d DatabaseType) IsValid() bool {
|
|
switch d {
|
|
case MongoDB, PostgreSQL, MySQL:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
type Database interface {
|
|
Connect(ctx context.Context) error
|
|
|
|
Close() error
|
|
|
|
InsertBatch(ctx context.Context, results []runner.Result) error
|
|
|
|
EnsureSchema(ctx context.Context) error
|
|
|
|
Type() DatabaseType
|
|
}
|
|
|
|
type databaseFactory func(cfg *Config) (Database, error)
|
|
|
|
var registry = make(map[DatabaseType]databaseFactory)
|
|
|
|
func Register(dbType DatabaseType, factory databaseFactory) {
|
|
registry[dbType] = factory
|
|
}
|
|
|
|
func NewDatabase(cfg *Config) (Database, error) {
|
|
if cfg == nil {
|
|
return nil, fmt.Errorf("database configuration is required")
|
|
}
|
|
|
|
if !cfg.Type.IsValid() {
|
|
return nil, fmt.Errorf("unsupported database type: %s", cfg.Type)
|
|
}
|
|
|
|
factory, ok := registry[cfg.Type]
|
|
if !ok {
|
|
return nil, fmt.Errorf("database type %s is not registered", cfg.Type)
|
|
}
|
|
|
|
return factory(cfg)
|
|
}
|
|
|
|
func SupportedDatabases() []DatabaseType {
|
|
return []DatabaseType{MongoDB, PostgreSQL, MySQL}
|
|
}
|