diff --git a/README.md b/README.md index dcdca1e41..fd0e41c91 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,11 @@ UPDATE: -ud, -update-template-dir string custom directory to install / update nuclei-templates -duc, -disable-update-check disable automatic nuclei/templates update check +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + STATISTICS: -stats display statistics about the running scan -sj, -stats-json display statistics in JSONL(ines) format diff --git a/README_CN.md b/README_CN.md index b52b219cc..22d013618 100644 --- a/README_CN.md +++ b/README_CN.md @@ -291,6 +291,11 @@ UNCOVER引擎: -ud, -update-template-dir string 指定模板目录 -duc, -disable-update-check 禁用nuclei程序与模板更新 +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + 统计： -stats 显示正在扫描的统计信息 -sj, -stats-json 将统计信息以JSONL格式输出到文件 diff --git a/README_ES.md b/README_ES.md index 68a6fa0ce..3959ee452 100644 --- a/README_ES.md +++ b/README_ES.md @@ -296,6 +296,11 @@ UPDATE: -ud, -update-template-dir string directorio personalizado para instalar/actualizar nuclei-templates -duc, -disable-update-check deshabilita la comprobación automática de actualizaciones de nuclei/templates +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + STATISTICS: -stats muestra estadísticas sobre el escaneo en ejecución -sj, -stats-json muestra estadísticas en formato JSONL(ines) diff --git a/README_ID.md b/README_ID.md index 632338b7c..848a17067 100644 --- a/README_ID.md +++ b/README_ID.md @@ -262,6 +262,11 @@ UPDATE: -ud, -update-template-dir string custom directory to install / update nuclei-templates -duc, -disable-update-check disable automatic nuclei/templates update check +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + STATISTICS: -stats display statistics about the running scan -sj, -stats-json display statistics in JSONL(ines) format diff --git a/README_KR.md b/README_KR.md index 65f24e1bc..6316cf1a8 100644 --- a/README_KR.md +++ b/README_KR.md @@ -261,6 +261,11 @@ UPDATE: -ud, -update-template-dir string nuclei-templates를 설치/업데이트할 사용자 지정 디렉토리 -duc, -disable-update-check 자동 nuclei/templates 업데이트 확인 비활성화 +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + STATISTICS: -stats 실행 중인 스캔에 대한 통계 표시 -sj, -stats-json JSONL(ines) 형식으로 통계 표시 diff --git a/README_PT-BR.md b/README_PT-BR.md index ad9cc17bf..4d8650394 100644 --- a/README_PT-BR.md +++ b/README_PT-BR.md @@ -296,6 +296,11 @@ UPDATE: -ud, -update-template-dir string diretório personalizado para instalar/atualizar os nuclei-templates -duc, -disable-update-check desativa a verificação automática de atualizações do nuclei/templates +HONEYPOT: + -hpd, -honeypot-detect detect potential honeypot hosts based on match concentration + -hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15) + -shp, -suppress-honeypot suppress output for flagged honeypot hosts + STATISTICS: -stats exibe estatísticas sobre o scan em execução -sj, -stats-json exibe estatísticas no formato JSONL(ines) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 09d8fe22c..f4cb0280c 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -487,6 +487,12 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"), ) + flagSet.CreateGroup("Honeypot", "Honeypot", + flagSet.BoolVarP(&options.HoneypotDetection, "honeypot-detect", "hpd", false, "detect potential honeypot hosts based on match concentration"), + flagSet.IntVarP(&options.HoneypotThreshold, "honeypot-threshold", "hpt", 15, "number of distinct template IDs required to flag a honeypot host"), + flagSet.BoolVarP(&options.SuppressHoneypotResults, "suppress-honeypot", "shp", false, "suppress output for flagged honeypot hosts"), + ) + flagSet.CreateGroup("stats", "Statistics", flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"), diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 35c5b4bbe..3b4ab1fb1 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -54,6 +54,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" @@ -97,6 +98,8 @@ type Runner struct { httpStats *outputstats.Tracker Logger *gologger.Logger + honeypotDetector *honeypotdetector.Detector + //general purpose temporary directory tmpDir string parser parser.Parser @@ -261,6 +264,12 @@ func New(options *types.Options) (*Runner, error) { } }() + // Initialize honeypot detector (opt-in) so results can be suppressed. + var hpDetector *honeypotdetector.Detector + if options.HoneypotDetection { + hpDetector = honeypotdetector.New(options.HoneypotThreshold) + } + // create the input provider and load the inputs inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir}) if err != nil { @@ -273,6 +282,10 @@ func New(options *types.Options) (*Runner, error) { if err != nil { return nil, errors.Wrap(err, "could not create output file") } + if hpDetector != nil { + outputWriter.SetHoneypotDetector(hpDetector) + runner.honeypotDetector = hpDetector + } // setup a proxy writer to automatically upload results to PDCP runner.output = runner.setupPDCPUpload(outputWriter) if options.HTTPStats { @@ -421,6 +434,10 @@ func (r *Runner) Close() { if r.output != nil { r.output.Close() } + + if r.honeypotDetector != nil { + r.Logger.Print().Msgf("%s\n", r.honeypotDetector.Summary()) + } if r.issuesClient != nil { r.issuesClient.Close() } diff --git a/pkg/output/output.go b/pkg/output/output.go index e18d26131..adebe867d 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -2,10 +2,12 @@ package output import ( "encoding/base64" + stderrors "errors" "fmt" "io" "log/slog" "maps" + "net" "os" "path/filepath" "regexp" @@ -27,6 +29,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/operators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector" protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr" @@ -38,6 +41,10 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) +// ErrHoneypotSuppressed is returned by the output writer when a match result is suppressed +// due to honeypot detection. +var ErrHoneypotSuppressed = stderrors.New("honeypot suppressed result") + // Writer is an interface which writes output to somewhere for nuclei events. type Writer interface { // Close closes the output writer interface @@ -65,6 +72,9 @@ type StandardWriter struct { timestamp bool noMetadata bool matcherStatus bool + honeypotDetector *honeypotdetector.Detector + suppressHoneypot bool + honeypotThreshold int mutex *sync.Mutex aurora aurora.Aurora outputFile io.WriteCloser @@ -265,21 +275,23 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { } writer := &StandardWriter{ - json: options.JSONL, - jsonReqResp: !options.OmitRawRequests, - noMetadata: options.NoMeta, - matcherStatus: options.MatcherStatus, - timestamp: options.Timestamp, - aurora: auroraColorizer, - mutex: &sync.Mutex{}, - outputFile: outputFile, - traceFile: traceOutput, - errorFile: errorOutput, - severityColors: colorizer.New(auroraColorizer), - storeResponse: options.StoreResponse, - storeResponseDir: options.StoreResponseDir, - omitTemplate: options.OmitTemplate, - KeysToRedact: options.Redact, + json: options.JSONL, + jsonReqResp: !options.OmitRawRequests, + noMetadata: options.NoMeta, + matcherStatus: options.MatcherStatus, + timestamp: options.Timestamp, + suppressHoneypot: options.SuppressHoneypotResults, + honeypotThreshold: options.HoneypotThreshold, + aurora: auroraColorizer, + mutex: &sync.Mutex{}, + outputFile: outputFile, + traceFile: traceOutput, + errorFile: errorOutput, + severityColors: colorizer.New(auroraColorizer), + storeResponse: options.StoreResponse, + storeResponseDir: options.StoreResponseDir, + omitTemplate: options.OmitTemplate, + KeysToRedact: options.Redact, } if v := os.Getenv("DISABLE_STDOUT"); v == "true" || v == "1" { @@ -289,6 +301,14 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { return writer, nil } +// SetHoneypotDetector attaches an initialized honeypot detector to the writer. +func (w *StandardWriter) SetHoneypotDetector(detector *honeypotdetector.Detector) { + w.honeypotDetector = detector + if detector != nil { + w.honeypotThreshold = detector.Threshold() + } +} + func (w *StandardWriter) ResultCount() int { return int(w.resultCount.Load()) } @@ -299,6 +319,29 @@ func (w *StandardWriter) Write(event *ResultEvent) error { return nil } + // Honeypot detection is performed only for successful matches. + if event.MatcherStatus && w.honeypotDetector != nil { + hostKey := event.URL + if hostKey == "" && event.Host != "" { + hostKey = event.Host + if event.Port != "" { + hostKey = net.JoinHostPort(event.Host, event.Port) + } + } + + if hostKey != "" { + justFlagged := w.honeypotDetector.RecordMatch(hostKey, event.TemplateID) + if justFlagged { + normalized := honeypotdetector.NormalizeHostKey(hostKey) + gologger.Warning().Msgf("Potential honeypot detected: %s (matched %d distinct templates)", normalized, w.honeypotThreshold) + } + + if w.suppressHoneypot && w.honeypotDetector.IsFlagged(hostKey) { + return ErrHoneypotSuppressed + } + } + } + // Enrich the result event with extra metadata on the template-path and url. if event.TemplatePath != "" { event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier) diff --git a/pkg/protocols/common/helpers/writer/writer.go b/pkg/protocols/common/helpers/writer/writer.go index 6666bde3c..19ef18c2d 100644 --- a/pkg/protocols/common/helpers/writer/writer.go +++ b/pkg/protocols/common/helpers/writer/writer.go @@ -1,6 +1,8 @@ package writer import ( + stderrors "errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/progress" @@ -8,7 +10,7 @@ import ( ) // WriteResult is a helper for writing results to the output -func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool { +func WriteResult(data *output.InternalWrappedEvent, out output.Writer, progress progress.Progress, issuesClient reporting.Client) bool { // Handle the case where no result found for the template. // In this case, we just show misc information about the failed // match for the template. @@ -17,18 +19,27 @@ func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progre } var matched bool for _, result := range data.Results { - if issuesClient != nil { + var suppressed bool + if err := out.Write(result); err != nil { + if stderrors.Is(err, output.ErrHoneypotSuppressed) { + suppressed = true + } else { + gologger.Warning().Msgf("Could not write output event: %s\n", err) + } + } + + // Only create issues when the result was not suppressed. + if issuesClient != nil && !suppressed { if err := issuesClient.CreateIssue(result); err != nil { gologger.Warning().Msgf("Could not create issue on tracker: %s", err) } } - if err := output.Write(result); err != nil { - gologger.Warning().Msgf("Could not write output event: %s\n", err) - } if !matched { matched = true } - progress.IncrementMatched() + if !suppressed { + progress.IncrementMatched() + } } return matched } diff --git a/pkg/protocols/common/honeypotdetector/honeypotdetector.go b/pkg/protocols/common/honeypotdetector/honeypotdetector.go new file mode 100644 index 000000000..687a5e3e7 --- /dev/null +++ b/pkg/protocols/common/honeypotdetector/honeypotdetector.go @@ -0,0 +1,190 @@ +package honeypotdetector + +import ( + "fmt" + "net" + "net/url" + "strings" + "sync" +) + +// Detector tracks honeypot likelihood by counting distinct template matches per normalized host. +// Once a host reaches the configured threshold, it becomes flagged. +type Detector struct { + threshold int + hosts sync.Map // map[normalizedHost]*hostState +} + +type hostState struct { + mu sync.Mutex + templateIDs map[string]struct{} + flagged bool +} + +// New creates a new honeypot detector. +func New(threshold int) *Detector { + if threshold <= 0 { + threshold = 1 + } + return &Detector{ + threshold: threshold, + } +} + +// Threshold returns the distinct template count required to flag a host. +func (d *Detector) Threshold() int { + if d == nil { + return 0 + } + return d.threshold +} + +// RecordMatch records a match for the given host and templateID. +// +// It returns true only when the host has just become flagged (i.e. crossed the threshold). +func (d *Detector) RecordMatch(host, templateID string) bool { + if d == nil { + return false + } + + normalizedHost := normalizeHostKey(host) + if normalizedHost == "" || templateID == "" { + return false + } + + stateAny, _ := d.hosts.LoadOrStore(normalizedHost, &hostState{ + templateIDs: make(map[string]struct{}), + }) + state := stateAny.(*hostState) + + state.mu.Lock() + defer state.mu.Unlock() + + if state.flagged { + return false + } + if _, ok := state.templateIDs[templateID]; ok { + return false + } + + state.templateIDs[templateID] = struct{}{} + if len(state.templateIDs) >= d.threshold { + state.flagged = true + state.templateIDs = nil + return true + } + return false +} + +// IsFlagged returns whether the given host is flagged. +func (d *Detector) IsFlagged(host string) bool { + if d == nil { + return false + } + + normalizedHost := normalizeHostKey(host) + if normalizedHost == "" { + return false + } + + stateAny, ok := d.hosts.Load(normalizedHost) + if !ok { + return false + } + state := stateAny.(*hostState) + + state.mu.Lock() + defer state.mu.Unlock() + return state.flagged +} + +// Summary returns a short string with the total number of flagged hosts. +func (d *Detector) Summary() string { + if d == nil { + return "honeypot-detected hosts: 0" + } + + var flagged int + d.hosts.Range(func(_, v any) bool { + state := v.(*hostState) + state.mu.Lock() + if state.flagged { + flagged++ + } + state.mu.Unlock() + return true + }) + + return fmt.Sprintf("honeypot-detected hosts: %d", flagged) +} + +// NormalizeHostKey normalizes host strings so different input formats map to the same key. +func NormalizeHostKey(input string) string { + return normalizeHostKey(input) +} + +func normalizeHostKey(input string) string { + s := strings.TrimSpace(input) + if s == "" { + return "" + } + + // Strip trailing slashes early. + s = strings.TrimRight(s, "/") + + // If an absolute URL is present, parse it to reliably extract host and optional port. + if strings.Contains(s, "://") { + u, err := url.Parse(s) + if err == nil && u != nil { + host := u.Hostname() + port := u.Port() + if host == "" { + return "" + } + host = normalizeHostWithoutPort(host) + if port != "" { + return net.JoinHostPort(host, port) + } + return host + } + // fall through if parsing fails + } + + // Remove any path suffix (we only care about the authority). + if idx := strings.IndexByte(s, '/'); idx >= 0 { + s = s[:idx] + } + + // If it looks like host:port (including bracketed IPv6), try SplitHostPort first. + if host, port, err := net.SplitHostPort(s); err == nil { + host = normalizeHostWithoutPort(host) + if port == "" { + return host + } + return net.JoinHostPort(host, port) + } + + // Handle bracketed IPv6 without port: [2001:db8::1] + if strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") { + host := strings.TrimSuffix(strings.TrimPrefix(s, "["), "]") + return normalizeHostWithoutPort(host) + } + + // Handle bare IPv6 or host without port. + return normalizeHostWithoutPort(s) +} + +func normalizeHostWithoutPort(host string) string { + h := strings.TrimSpace(host) + if h == "" { + return "" + } + h = strings.TrimPrefix(h, "[") + h = strings.TrimSuffix(h, "]") + h = strings.ToLower(h) + + if ip := net.ParseIP(h); ip != nil { + return ip.String() + } + return h +} diff --git a/pkg/protocols/common/honeypotdetector/honeypotdetector_test.go b/pkg/protocols/common/honeypotdetector/honeypotdetector_test.go new file mode 100644 index 000000000..552c9395f --- /dev/null +++ b/pkg/protocols/common/honeypotdetector/honeypotdetector_test.go @@ -0,0 +1,108 @@ +package honeypotdetector + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestThresholdBoundary(t *testing.T) { + t.Parallel() + + d := New(2) + host := "example.com" + + require.False(t, d.RecordMatch(host, "t1"), "flagging should not happen at N-1 matches") + require.True(t, d.RecordMatch(host, "t2"), "flagging should happen at exactly N distinct template IDs") + require.True(t, d.IsFlagged(host)) + + // Once flagged, additional distinct matches should not re-trigger the boundary condition. + require.False(t, d.RecordMatch(host, "t3")) +} + +func TestDeduplication(t *testing.T) { + t.Parallel() + + d := New(2) + host := "example.com" + + require.False(t, d.RecordMatch(host, "t1")) + require.False(t, d.RecordMatch(host, "t1"), "same templateID on same host must count once") + require.True(t, d.RecordMatch(host, "t2")) + require.True(t, d.IsFlagged(host)) +} + +func TestHostIsolation(t *testing.T) { + t.Parallel() + + d := New(2) + + require.False(t, d.RecordMatch("example-a.com", "t1")) + require.False(t, d.RecordMatch("example-b.com", "t1")) + require.True(t, d.RecordMatch("example-a.com", "t2"), "host A should be flagged independently") + + require.True(t, d.IsFlagged("example-a.com")) + require.False(t, d.IsFlagged("example-b.com")) +} + +func TestConcurrentAccess(t *testing.T) { + t.Parallel() + + const ( + threshold = 10 + goroutines = 100 + ) + + d := New(threshold) + host := "example.com" + + var justFlaggedCount atomic.Int32 + var wg sync.WaitGroup + wg.Add(goroutines) + + for i := 0; i < goroutines; i++ { + i := i + go func() { + defer wg.Done() + if d.RecordMatch(host, fmt.Sprintf("t-%d", i)) { + justFlaggedCount.Add(1) + } + }() + } + + wg.Wait() + + require.True(t, d.IsFlagged(host)) + require.Equal(t, int32(1), justFlaggedCount.Load(), "exactly one goroutine should trigger the boundary") +} + +func TestHostNormalization(t *testing.T) { + t.Parallel() + + d := New(2) + + // Scheme + trailing slash must be stripped so these map to the same normalized host. + require.False(t, d.RecordMatch("https://example.com/", "t1")) + require.True(t, d.RecordMatch("http://example.com", "t2")) + require.True(t, d.IsFlagged("example.com")) + + // Explicit port should keep it distinct (example.com and example.com:443 are different keys). + d2 := New(1) + require.True(t, d2.RecordMatch("example.com:443", "t1")) + require.True(t, d2.IsFlagged("example.com:443")) + require.False(t, d2.IsFlagged("example.com"), "host without explicit port must not share the same key") + + // IPv6 should be canonicalized, and bracketed host:port should keep bracketed form when port is present. + d3 := New(1) + require.True(t, d3.RecordMatch("http://[2001:db8::1]/", "t1")) + require.True(t, d3.IsFlagged("2001:db8::1")) + require.True(t, d3.IsFlagged("[2001:db8::1]")) + + d4 := New(1) + require.True(t, d4.RecordMatch("[2001:db8::1]:443", "t1")) + require.True(t, d4.IsFlagged("[2001:db8::1]:443")) + require.False(t, d4.IsFlagged("2001:db8::1"), "IPv6 with explicit port must not share the same key as IPv6 without port") +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 596f76d06..80ba74812 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -258,6 +258,13 @@ type Options struct { Stdin bool // StopAtFirstMatch stops processing template at first full match (this may break chained requests) StopAtFirstMatch bool + + // HoneypotDetection enables detection of potential honeypots based on match concentration. + HoneypotDetection bool + // HoneypotThreshold is the number of distinct template IDs required to flag a host as a potential honeypot. + HoneypotThreshold int + // SuppressHoneypotResults suppresses output writing for flagged honeypot hosts. + SuppressHoneypotResults bool // Stream the input without sorting Stream bool // NoMeta disables display of metadata for the matches @@ -588,6 +595,9 @@ func (options *Options) Copy() *Options { HangMonitor: options.HangMonitor, Stdin: options.Stdin, StopAtFirstMatch: options.StopAtFirstMatch, + HoneypotDetection: options.HoneypotDetection, + HoneypotThreshold: options.HoneypotThreshold, + SuppressHoneypotResults: options.SuppressHoneypotResults, Stream: options.Stream, NoMeta: options.NoMeta, Timestamp: options.Timestamp,