mirror of
https://github.com/projectdiscovery/nuclei
synced 2026-06-08 16:50:47 +00:00
Merge pull request #7277 from HarshadaGawas05/feat/honeypot-detection
feat: add honeypot detection to reduce scan noise (#6403)
This commit is contained in:
@@ -338,6 +338,11 @@ UPDATE:
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats display statistics about the running scan
|
||||
-sj, -stats-json display statistics in JSONL(ines) format
|
||||
|
||||
@@ -291,6 +291,11 @@ UNCOVER引擎:
|
||||
-ud, -update-template-dir string 指定模板目录
|
||||
-duc, -disable-update-check 禁用nuclei程序与模板更新
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
统计:
|
||||
-stats 显示正在扫描的统计信息
|
||||
-sj, -stats-json 将统计信息以JSONL格式输出到文件
|
||||
|
||||
@@ -296,6 +296,11 @@ UPDATE:
|
||||
-ud, -update-template-dir string directorio personalizado para instalar/actualizar nuclei-templates
|
||||
-duc, -disable-update-check deshabilita la comprobación automática de actualizaciones de nuclei/templates
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats muestra estadísticas sobre el escaneo en ejecución
|
||||
-sj, -stats-json muestra estadísticas en formato JSONL(ines)
|
||||
|
||||
@@ -262,6 +262,11 @@ UPDATE:
|
||||
-ud, -update-template-dir string custom directory to install / update nuclei-templates
|
||||
-duc, -disable-update-check disable automatic nuclei/templates update check
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats display statistics about the running scan
|
||||
-sj, -stats-json display statistics in JSONL(ines) format
|
||||
|
||||
@@ -261,6 +261,11 @@ UPDATE:
|
||||
-ud, -update-template-dir string nuclei-templates를 설치/업데이트할 사용자 지정 디렉토리
|
||||
-duc, -disable-update-check 자동 nuclei/templates 업데이트 확인 비활성화
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats 실행 중인 스캔에 대한 통계 표시
|
||||
-sj, -stats-json JSONL(ines) 형식으로 통계 표시
|
||||
|
||||
@@ -296,6 +296,11 @@ UPDATE:
|
||||
-ud, -update-template-dir string diretório personalizado para instalar/atualizar os nuclei-templates
|
||||
-duc, -disable-update-check desativa a verificação automática de atualizações do nuclei/templates
|
||||
|
||||
HONEYPOT:
|
||||
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
|
||||
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
|
||||
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
|
||||
|
||||
STATISTICS:
|
||||
-stats exibe estatísticas sobre o scan em execução
|
||||
-sj, -stats-json exibe estatísticas no formato JSONL(ines)
|
||||
|
||||
@@ -487,6 +487,12 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("Honeypot", "Honeypot",
|
||||
flagSet.BoolVarP(&options.HoneypotDetection, "honeypot-detect", "hpd", false, "detect potential honeypot hosts based on match concentration"),
|
||||
flagSet.IntVarP(&options.HoneypotThreshold, "honeypot-threshold", "hpt", 15, "number of distinct template IDs required to flag a honeypot host"),
|
||||
flagSet.BoolVarP(&options.SuppressHoneypotResults, "suppress-honeypot", "shp", false, "suppress output for flagged honeypot hosts"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("stats", "Statistics",
|
||||
flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
|
||||
flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
|
||||
|
||||
@@ -54,6 +54,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
@@ -97,6 +98,8 @@ type Runner struct {
|
||||
httpStats *outputstats.Tracker
|
||||
Logger *gologger.Logger
|
||||
|
||||
honeypotDetector *honeypotdetector.Detector
|
||||
|
||||
//general purpose temporary directory
|
||||
tmpDir string
|
||||
parser parser.Parser
|
||||
@@ -261,6 +264,12 @@ func New(options *types.Options) (*Runner, error) {
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize honeypot detector (opt-in) so results can be suppressed.
|
||||
var hpDetector *honeypotdetector.Detector
|
||||
if options.HoneypotDetection {
|
||||
hpDetector = honeypotdetector.New(options.HoneypotThreshold)
|
||||
}
|
||||
|
||||
// create the input provider and load the inputs
|
||||
inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir})
|
||||
if err != nil {
|
||||
@@ -273,6 +282,10 @@ func New(options *types.Options) (*Runner, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not create output file")
|
||||
}
|
||||
if hpDetector != nil {
|
||||
outputWriter.SetHoneypotDetector(hpDetector)
|
||||
runner.honeypotDetector = hpDetector
|
||||
}
|
||||
// setup a proxy writer to automatically upload results to PDCP
|
||||
runner.output = runner.setupPDCPUpload(outputWriter)
|
||||
if options.HTTPStats {
|
||||
@@ -421,6 +434,10 @@ func (r *Runner) Close() {
|
||||
if r.output != nil {
|
||||
r.output.Close()
|
||||
}
|
||||
|
||||
if r.honeypotDetector != nil {
|
||||
r.Logger.Print().Msgf("%s\n", r.honeypotDetector.Summary())
|
||||
}
|
||||
if r.issuesClient != nil {
|
||||
r.issuesClient.Close()
|
||||
}
|
||||
|
||||
+58
-15
@@ -2,10 +2,12 @@ package output
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@@ -27,6 +29,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
|
||||
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
|
||||
@@ -38,6 +41,10 @@ import (
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
// ErrHoneypotSuppressed is returned by the output writer when a match result is suppressed
|
||||
// due to honeypot detection.
|
||||
var ErrHoneypotSuppressed = stderrors.New("honeypot suppressed result")
|
||||
|
||||
// Writer is an interface which writes output to somewhere for nuclei events.
|
||||
type Writer interface {
|
||||
// Close closes the output writer interface
|
||||
@@ -65,6 +72,9 @@ type StandardWriter struct {
|
||||
timestamp bool
|
||||
noMetadata bool
|
||||
matcherStatus bool
|
||||
honeypotDetector *honeypotdetector.Detector
|
||||
suppressHoneypot bool
|
||||
honeypotThreshold int
|
||||
mutex *sync.Mutex
|
||||
aurora aurora.Aurora
|
||||
outputFile io.WriteCloser
|
||||
@@ -265,21 +275,23 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
|
||||
}
|
||||
|
||||
writer := &StandardWriter{
|
||||
json: options.JSONL,
|
||||
jsonReqResp: !options.OmitRawRequests,
|
||||
noMetadata: options.NoMeta,
|
||||
matcherStatus: options.MatcherStatus,
|
||||
timestamp: options.Timestamp,
|
||||
aurora: auroraColorizer,
|
||||
mutex: &sync.Mutex{},
|
||||
outputFile: outputFile,
|
||||
traceFile: traceOutput,
|
||||
errorFile: errorOutput,
|
||||
severityColors: colorizer.New(auroraColorizer),
|
||||
storeResponse: options.StoreResponse,
|
||||
storeResponseDir: options.StoreResponseDir,
|
||||
omitTemplate: options.OmitTemplate,
|
||||
KeysToRedact: options.Redact,
|
||||
json: options.JSONL,
|
||||
jsonReqResp: !options.OmitRawRequests,
|
||||
noMetadata: options.NoMeta,
|
||||
matcherStatus: options.MatcherStatus,
|
||||
timestamp: options.Timestamp,
|
||||
suppressHoneypot: options.SuppressHoneypotResults,
|
||||
honeypotThreshold: options.HoneypotThreshold,
|
||||
aurora: auroraColorizer,
|
||||
mutex: &sync.Mutex{},
|
||||
outputFile: outputFile,
|
||||
traceFile: traceOutput,
|
||||
errorFile: errorOutput,
|
||||
severityColors: colorizer.New(auroraColorizer),
|
||||
storeResponse: options.StoreResponse,
|
||||
storeResponseDir: options.StoreResponseDir,
|
||||
omitTemplate: options.OmitTemplate,
|
||||
KeysToRedact: options.Redact,
|
||||
}
|
||||
|
||||
if v := os.Getenv("DISABLE_STDOUT"); v == "true" || v == "1" {
|
||||
@@ -289,6 +301,14 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
// SetHoneypotDetector attaches an initialized honeypot detector to the writer.
|
||||
func (w *StandardWriter) SetHoneypotDetector(detector *honeypotdetector.Detector) {
|
||||
w.honeypotDetector = detector
|
||||
if detector != nil {
|
||||
w.honeypotThreshold = detector.Threshold()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *StandardWriter) ResultCount() int {
|
||||
return int(w.resultCount.Load())
|
||||
}
|
||||
@@ -299,6 +319,29 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Honeypot detection is performed only for successful matches.
|
||||
if event.MatcherStatus && w.honeypotDetector != nil {
|
||||
hostKey := event.URL
|
||||
if hostKey == "" && event.Host != "" {
|
||||
hostKey = event.Host
|
||||
if event.Port != "" {
|
||||
hostKey = net.JoinHostPort(event.Host, event.Port)
|
||||
}
|
||||
}
|
||||
|
||||
if hostKey != "" {
|
||||
justFlagged := w.honeypotDetector.RecordMatch(hostKey, event.TemplateID)
|
||||
if justFlagged {
|
||||
normalized := honeypotdetector.NormalizeHostKey(hostKey)
|
||||
gologger.Warning().Msgf("Potential honeypot detected: %s (matched %d distinct templates)", normalized, w.honeypotThreshold)
|
||||
}
|
||||
|
||||
if w.suppressHoneypot && w.honeypotDetector.IsFlagged(hostKey) {
|
||||
return ErrHoneypotSuppressed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Enrich the result event with extra metadata on the template-path and url.
|
||||
if event.TemplatePath != "" {
|
||||
event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package writer
|
||||
|
||||
import (
|
||||
stderrors "errors"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
|
||||
@@ -8,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// WriteResult is a helper for writing results to the output
|
||||
func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
|
||||
func WriteResult(data *output.InternalWrappedEvent, out output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
|
||||
// Handle the case where no result found for the template.
|
||||
// In this case, we just show misc information about the failed
|
||||
// match for the template.
|
||||
@@ -17,18 +19,27 @@ func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progre
|
||||
}
|
||||
var matched bool
|
||||
for _, result := range data.Results {
|
||||
if issuesClient != nil {
|
||||
var suppressed bool
|
||||
if err := out.Write(result); err != nil {
|
||||
if stderrors.Is(err, output.ErrHoneypotSuppressed) {
|
||||
suppressed = true
|
||||
} else {
|
||||
gologger.Warning().Msgf("Could not write output event: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Only create issues when the result was not suppressed.
|
||||
if issuesClient != nil && !suppressed {
|
||||
if err := issuesClient.CreateIssue(result); err != nil {
|
||||
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
|
||||
}
|
||||
}
|
||||
if err := output.Write(result); err != nil {
|
||||
gologger.Warning().Msgf("Could not write output event: %s\n", err)
|
||||
}
|
||||
if !matched {
|
||||
matched = true
|
||||
}
|
||||
progress.IncrementMatched()
|
||||
if !suppressed {
|
||||
progress.IncrementMatched()
|
||||
}
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package honeypotdetector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Detector tracks honeypot likelihood by counting distinct template matches per normalized host.
|
||||
// Once a host reaches the configured threshold, it becomes flagged.
|
||||
type Detector struct {
|
||||
threshold int
|
||||
hosts sync.Map // map[normalizedHost]*hostState
|
||||
}
|
||||
|
||||
type hostState struct {
|
||||
mu sync.Mutex
|
||||
templateIDs map[string]struct{}
|
||||
flagged bool
|
||||
}
|
||||
|
||||
// New creates a new honeypot detector.
|
||||
func New(threshold int) *Detector {
|
||||
if threshold <= 0 {
|
||||
threshold = 1
|
||||
}
|
||||
return &Detector{
|
||||
threshold: threshold,
|
||||
}
|
||||
}
|
||||
|
||||
// Threshold returns the distinct template count required to flag a host.
|
||||
func (d *Detector) Threshold() int {
|
||||
if d == nil {
|
||||
return 0
|
||||
}
|
||||
return d.threshold
|
||||
}
|
||||
|
||||
// RecordMatch records a match for the given host and templateID.
|
||||
//
|
||||
// It returns true only when the host has just become flagged (i.e. crossed the threshold).
|
||||
func (d *Detector) RecordMatch(host, templateID string) bool {
|
||||
if d == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
normalizedHost := normalizeHostKey(host)
|
||||
if normalizedHost == "" || templateID == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
stateAny, _ := d.hosts.LoadOrStore(normalizedHost, &hostState{
|
||||
templateIDs: make(map[string]struct{}),
|
||||
})
|
||||
state := stateAny.(*hostState)
|
||||
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
|
||||
if state.flagged {
|
||||
return false
|
||||
}
|
||||
if _, ok := state.templateIDs[templateID]; ok {
|
||||
return false
|
||||
}
|
||||
|
||||
state.templateIDs[templateID] = struct{}{}
|
||||
if len(state.templateIDs) >= d.threshold {
|
||||
state.flagged = true
|
||||
state.templateIDs = nil
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsFlagged returns whether the given host is flagged.
|
||||
func (d *Detector) IsFlagged(host string) bool {
|
||||
if d == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
normalizedHost := normalizeHostKey(host)
|
||||
if normalizedHost == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
stateAny, ok := d.hosts.Load(normalizedHost)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
state := stateAny.(*hostState)
|
||||
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
return state.flagged
|
||||
}
|
||||
|
||||
// Summary returns a short string with the total number of flagged hosts.
|
||||
func (d *Detector) Summary() string {
|
||||
if d == nil {
|
||||
return "honeypot-detected hosts: 0"
|
||||
}
|
||||
|
||||
var flagged int
|
||||
d.hosts.Range(func(_, v any) bool {
|
||||
state := v.(*hostState)
|
||||
state.mu.Lock()
|
||||
if state.flagged {
|
||||
flagged++
|
||||
}
|
||||
state.mu.Unlock()
|
||||
return true
|
||||
})
|
||||
|
||||
return fmt.Sprintf("honeypot-detected hosts: %d", flagged)
|
||||
}
|
||||
|
||||
// NormalizeHostKey normalizes host strings so different input formats map to the same key.
|
||||
func NormalizeHostKey(input string) string {
|
||||
return normalizeHostKey(input)
|
||||
}
|
||||
|
||||
func normalizeHostKey(input string) string {
|
||||
s := strings.TrimSpace(input)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Strip trailing slashes early.
|
||||
s = strings.TrimRight(s, "/")
|
||||
|
||||
// If an absolute URL is present, parse it to reliably extract host and optional port.
|
||||
if strings.Contains(s, "://") {
|
||||
u, err := url.Parse(s)
|
||||
if err == nil && u != nil {
|
||||
host := u.Hostname()
|
||||
port := u.Port()
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
host = normalizeHostWithoutPort(host)
|
||||
if port != "" {
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
return host
|
||||
}
|
||||
// fall through if parsing fails
|
||||
}
|
||||
|
||||
// Remove any path suffix (we only care about the authority).
|
||||
if idx := strings.IndexByte(s, '/'); idx >= 0 {
|
||||
s = s[:idx]
|
||||
}
|
||||
|
||||
// If it looks like host:port (including bracketed IPv6), try SplitHostPort first.
|
||||
if host, port, err := net.SplitHostPort(s); err == nil {
|
||||
host = normalizeHostWithoutPort(host)
|
||||
if port == "" {
|
||||
return host
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
// Handle bracketed IPv6 without port: [2001:db8::1]
|
||||
if strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") {
|
||||
host := strings.TrimSuffix(strings.TrimPrefix(s, "["), "]")
|
||||
return normalizeHostWithoutPort(host)
|
||||
}
|
||||
|
||||
// Handle bare IPv6 or host without port.
|
||||
return normalizeHostWithoutPort(s)
|
||||
}
|
||||
|
||||
func normalizeHostWithoutPort(host string) string {
|
||||
h := strings.TrimSpace(host)
|
||||
if h == "" {
|
||||
return ""
|
||||
}
|
||||
h = strings.TrimPrefix(h, "[")
|
||||
h = strings.TrimSuffix(h, "]")
|
||||
h = strings.ToLower(h)
|
||||
|
||||
if ip := net.ParseIP(h); ip != nil {
|
||||
return ip.String()
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package honeypotdetector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestThresholdBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
d := New(2)
|
||||
host := "example.com"
|
||||
|
||||
require.False(t, d.RecordMatch(host, "t1"), "flagging should not happen at N-1 matches")
|
||||
require.True(t, d.RecordMatch(host, "t2"), "flagging should happen at exactly N distinct template IDs")
|
||||
require.True(t, d.IsFlagged(host))
|
||||
|
||||
// Once flagged, additional distinct matches should not re-trigger the boundary condition.
|
||||
require.False(t, d.RecordMatch(host, "t3"))
|
||||
}
|
||||
|
||||
func TestDeduplication(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
d := New(2)
|
||||
host := "example.com"
|
||||
|
||||
require.False(t, d.RecordMatch(host, "t1"))
|
||||
require.False(t, d.RecordMatch(host, "t1"), "same templateID on same host must count once")
|
||||
require.True(t, d.RecordMatch(host, "t2"))
|
||||
require.True(t, d.IsFlagged(host))
|
||||
}
|
||||
|
||||
func TestHostIsolation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
d := New(2)
|
||||
|
||||
require.False(t, d.RecordMatch("example-a.com", "t1"))
|
||||
require.False(t, d.RecordMatch("example-b.com", "t1"))
|
||||
require.True(t, d.RecordMatch("example-a.com", "t2"), "host A should be flagged independently")
|
||||
|
||||
require.True(t, d.IsFlagged("example-a.com"))
|
||||
require.False(t, d.IsFlagged("example-b.com"))
|
||||
}
|
||||
|
||||
func TestConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
threshold = 10
|
||||
goroutines = 100
|
||||
)
|
||||
|
||||
d := New(threshold)
|
||||
host := "example.com"
|
||||
|
||||
var justFlaggedCount atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
|
||||
for i := 0; i < goroutines; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if d.RecordMatch(host, fmt.Sprintf("t-%d", i)) {
|
||||
justFlaggedCount.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
require.True(t, d.IsFlagged(host))
|
||||
require.Equal(t, int32(1), justFlaggedCount.Load(), "exactly one goroutine should trigger the boundary")
|
||||
}
|
||||
|
||||
func TestHostNormalization(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
d := New(2)
|
||||
|
||||
// Scheme + trailing slash must be stripped so these map to the same normalized host.
|
||||
require.False(t, d.RecordMatch("https://example.com/", "t1"))
|
||||
require.True(t, d.RecordMatch("http://example.com", "t2"))
|
||||
require.True(t, d.IsFlagged("example.com"))
|
||||
|
||||
// Explicit port should keep it distinct (example.com and example.com:443 are different keys).
|
||||
d2 := New(1)
|
||||
require.True(t, d2.RecordMatch("example.com:443", "t1"))
|
||||
require.True(t, d2.IsFlagged("example.com:443"))
|
||||
require.False(t, d2.IsFlagged("example.com"), "host without explicit port must not share the same key")
|
||||
|
||||
// IPv6 should be canonicalized, and bracketed host:port should keep bracketed form when port is present.
|
||||
d3 := New(1)
|
||||
require.True(t, d3.RecordMatch("http://[2001:db8::1]/", "t1"))
|
||||
require.True(t, d3.IsFlagged("2001:db8::1"))
|
||||
require.True(t, d3.IsFlagged("[2001:db8::1]"))
|
||||
|
||||
d4 := New(1)
|
||||
require.True(t, d4.RecordMatch("[2001:db8::1]:443", "t1"))
|
||||
require.True(t, d4.IsFlagged("[2001:db8::1]:443"))
|
||||
require.False(t, d4.IsFlagged("2001:db8::1"), "IPv6 with explicit port must not share the same key as IPv6 without port")
|
||||
}
|
||||
@@ -258,6 +258,13 @@ type Options struct {
|
||||
Stdin bool
|
||||
// StopAtFirstMatch stops processing template at first full match (this may break chained requests)
|
||||
StopAtFirstMatch bool
|
||||
|
||||
// HoneypotDetection enables detection of potential honeypots based on match concentration.
|
||||
HoneypotDetection bool
|
||||
// HoneypotThreshold is the number of distinct template IDs required to flag a host as a potential honeypot.
|
||||
HoneypotThreshold int
|
||||
// SuppressHoneypotResults suppresses output writing for flagged honeypot hosts.
|
||||
SuppressHoneypotResults bool
|
||||
// Stream the input without sorting
|
||||
Stream bool
|
||||
// NoMeta disables display of metadata for the matches
|
||||
@@ -588,6 +595,9 @@ func (options *Options) Copy() *Options {
|
||||
HangMonitor: options.HangMonitor,
|
||||
Stdin: options.Stdin,
|
||||
StopAtFirstMatch: options.StopAtFirstMatch,
|
||||
HoneypotDetection: options.HoneypotDetection,
|
||||
HoneypotThreshold: options.HoneypotThreshold,
|
||||
SuppressHoneypotResults: options.SuppressHoneypotResults,
|
||||
Stream: options.Stream,
|
||||
NoMeta: options.NoMeta,
|
||||
Timestamp: options.Timestamp,
|
||||
|
||||
Reference in New Issue
Block a user