Merge pull request #7277 from HarshadaGawas05/feat/honeypot-detection

feat: add honeypot detection to reduce scan noise (#6403)
This commit is contained in:
Mzack9999
2026-03-20 17:09:58 +01:00
committed by GitHub
13 changed files with 436 additions and 21 deletions
+5
View File
@@ -338,6 +338,11 @@ UPDATE:
-ud, -update-template-dir string custom directory to install / update nuclei-templates
-duc, -disable-update-check disable automatic nuclei/templates update check
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
STATISTICS:
-stats display statistics about the running scan
-sj, -stats-json display statistics in JSONL(ines) format
+5
View File
@@ -291,6 +291,11 @@ UNCOVER引擎:
-ud, -update-template-dir string 指定模板目录
-duc, -disable-update-check 禁用nuclei程序与模板更新
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
统计:
-stats 显示正在扫描的统计信息
-sj, -stats-json 将统计信息以JSONL格式输出到文件
+5
View File
@@ -296,6 +296,11 @@ UPDATE:
-ud, -update-template-dir string directorio personalizado para instalar/actualizar nuclei-templates
-duc, -disable-update-check deshabilita la comprobación automática de actualizaciones de nuclei/templates
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
STATISTICS:
-stats muestra estadísticas sobre el escaneo en ejecución
-sj, -stats-json muestra estadísticas en formato JSONL(ines)
+5
View File
@@ -262,6 +262,11 @@ UPDATE:
-ud, -update-template-dir string custom directory to install / update nuclei-templates
-duc, -disable-update-check disable automatic nuclei/templates update check
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
STATISTICS:
-stats display statistics about the running scan
-sj, -stats-json display statistics in JSONL(ines) format
+5
View File
@@ -261,6 +261,11 @@ UPDATE:
-ud, -update-template-dir string nuclei-templates를 설치/업데이트할 사용자 지정 디렉토리
-duc, -disable-update-check 자동 nuclei/templates 업데이트 확인 비활성화
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
STATISTICS:
-stats 실행 중인 스캔에 대한 통계 표시
-sj, -stats-json JSONL(ines) 형식으로 통계 표시
+5
View File
@@ -296,6 +296,11 @@ UPDATE:
-ud, -update-template-dir string diretório personalizado para instalar/atualizar os nuclei-templates
-duc, -disable-update-check desativa a verificação automática de atualizações do nuclei/templates
HONEYPOT:
-hpd, -honeypot-detect detect potential honeypot hosts based on match concentration
-hpt, -honeypot-threshold int number of distinct template IDs required to flag a honeypot host (default 15)
-shp, -suppress-honeypot suppress output for flagged honeypot hosts
STATISTICS:
-stats exibe estatísticas sobre o scan em execução
-sj, -stats-json exibe estatísticas no formato JSONL(ines)
+6
View File
@@ -487,6 +487,12 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CallbackVarP(disableUpdatesCallback, "disable-update-check", "duc", "disable automatic nuclei/templates update check"),
)
flagSet.CreateGroup("Honeypot", "Honeypot",
flagSet.BoolVarP(&options.HoneypotDetection, "honeypot-detect", "hpd", false, "detect potential honeypot hosts based on match concentration"),
flagSet.IntVarP(&options.HoneypotThreshold, "honeypot-threshold", "hpt", 15, "number of distinct template IDs required to flag a honeypot host"),
flagSet.BoolVarP(&options.SuppressHoneypotResults, "suppress-honeypot", "shp", false, "suppress output for flagged honeypot hosts"),
)
flagSet.CreateGroup("stats", "Statistics",
flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"),
flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "display statistics in JSONL(ines) format"),
+17
View File
@@ -54,6 +54,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
@@ -97,6 +98,8 @@ type Runner struct {
httpStats *outputstats.Tracker
Logger *gologger.Logger
honeypotDetector *honeypotdetector.Detector
//general purpose temporary directory
tmpDir string
parser parser.Parser
@@ -261,6 +264,12 @@ func New(options *types.Options) (*Runner, error) {
}
}()
// Initialize honeypot detector (opt-in) so results can be suppressed.
var hpDetector *honeypotdetector.Detector
if options.HoneypotDetection {
hpDetector = honeypotdetector.New(options.HoneypotThreshold)
}
// create the input provider and load the inputs
inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir})
if err != nil {
@@ -273,6 +282,10 @@ func New(options *types.Options) (*Runner, error) {
if err != nil {
return nil, errors.Wrap(err, "could not create output file")
}
if hpDetector != nil {
outputWriter.SetHoneypotDetector(hpDetector)
runner.honeypotDetector = hpDetector
}
// setup a proxy writer to automatically upload results to PDCP
runner.output = runner.setupPDCPUpload(outputWriter)
if options.HTTPStats {
@@ -421,6 +434,10 @@ func (r *Runner) Close() {
if r.output != nil {
r.output.Close()
}
if r.honeypotDetector != nil {
r.Logger.Print().Msgf("%s\n", r.honeypotDetector.Summary())
}
if r.issuesClient != nil {
r.issuesClient.Close()
}
+58 -15
View File
@@ -2,10 +2,12 @@ package output
import (
"encoding/base64"
stderrors "errors"
"fmt"
"io"
"log/slog"
"maps"
"net"
"os"
"path/filepath"
"regexp"
@@ -27,6 +29,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
@@ -38,6 +41,10 @@ import (
urlutil "github.com/projectdiscovery/utils/url"
)
// ErrHoneypotSuppressed is returned by the output writer when a match result is suppressed
// due to honeypot detection.
var ErrHoneypotSuppressed = stderrors.New("honeypot suppressed result")
// Writer is an interface which writes output to somewhere for nuclei events.
type Writer interface {
// Close closes the output writer interface
@@ -65,6 +72,9 @@ type StandardWriter struct {
timestamp bool
noMetadata bool
matcherStatus bool
honeypotDetector *honeypotdetector.Detector
suppressHoneypot bool
honeypotThreshold int
mutex *sync.Mutex
aurora aurora.Aurora
outputFile io.WriteCloser
@@ -265,21 +275,23 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
}
writer := &StandardWriter{
json: options.JSONL,
jsonReqResp: !options.OmitRawRequests,
noMetadata: options.NoMeta,
matcherStatus: options.MatcherStatus,
timestamp: options.Timestamp,
aurora: auroraColorizer,
mutex: &sync.Mutex{},
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
omitTemplate: options.OmitTemplate,
KeysToRedact: options.Redact,
json: options.JSONL,
jsonReqResp: !options.OmitRawRequests,
noMetadata: options.NoMeta,
matcherStatus: options.MatcherStatus,
timestamp: options.Timestamp,
suppressHoneypot: options.SuppressHoneypotResults,
honeypotThreshold: options.HoneypotThreshold,
aurora: auroraColorizer,
mutex: &sync.Mutex{},
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
omitTemplate: options.OmitTemplate,
KeysToRedact: options.Redact,
}
if v := os.Getenv("DISABLE_STDOUT"); v == "true" || v == "1" {
@@ -289,6 +301,14 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
return writer, nil
}
// SetHoneypotDetector attaches an initialized honeypot detector to the writer.
func (w *StandardWriter) SetHoneypotDetector(detector *honeypotdetector.Detector) {
w.honeypotDetector = detector
if detector != nil {
w.honeypotThreshold = detector.Threshold()
}
}
func (w *StandardWriter) ResultCount() int {
return int(w.resultCount.Load())
}
@@ -299,6 +319,29 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
return nil
}
// Honeypot detection is performed only for successful matches.
if event.MatcherStatus && w.honeypotDetector != nil {
hostKey := event.URL
if hostKey == "" && event.Host != "" {
hostKey = event.Host
if event.Port != "" {
hostKey = net.JoinHostPort(event.Host, event.Port)
}
}
if hostKey != "" {
justFlagged := w.honeypotDetector.RecordMatch(hostKey, event.TemplateID)
if justFlagged {
normalized := honeypotdetector.NormalizeHostKey(hostKey)
gologger.Warning().Msgf("Potential honeypot detected: %s (matched %d distinct templates)", normalized, w.honeypotThreshold)
}
if w.suppressHoneypot && w.honeypotDetector.IsFlagged(hostKey) {
return ErrHoneypotSuppressed
}
}
}
// Enrich the result event with extra metadata on the template-path and url.
if event.TemplatePath != "" {
event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath), types.ToString(event.TemplateID), event.TemplateVerifier)
+17 -6
View File
@@ -1,6 +1,8 @@
package writer
import (
stderrors "errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
@@ -8,7 +10,7 @@ import (
)
// WriteResult is a helper for writing results to the output
func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
func WriteResult(data *output.InternalWrappedEvent, out output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
// Handle the case where no result found for the template.
// In this case, we just show misc information about the failed
// match for the template.
@@ -17,18 +19,27 @@ func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progre
}
var matched bool
for _, result := range data.Results {
if issuesClient != nil {
var suppressed bool
if err := out.Write(result); err != nil {
if stderrors.Is(err, output.ErrHoneypotSuppressed) {
suppressed = true
} else {
gologger.Warning().Msgf("Could not write output event: %s\n", err)
}
}
// Only create issues when the result was not suppressed.
if issuesClient != nil && !suppressed {
if err := issuesClient.CreateIssue(result); err != nil {
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
}
}
if err := output.Write(result); err != nil {
gologger.Warning().Msgf("Could not write output event: %s\n", err)
}
if !matched {
matched = true
}
progress.IncrementMatched()
if !suppressed {
progress.IncrementMatched()
}
}
return matched
}
@@ -0,0 +1,190 @@
package honeypotdetector
import (
"fmt"
"net"
"net/url"
"strings"
"sync"
)
// Detector tracks honeypot likelihood by counting distinct template matches per normalized host.
// Once a host reaches the configured threshold, it becomes flagged.
type Detector struct {
threshold int
hosts sync.Map // map[normalizedHost]*hostState
}
type hostState struct {
mu sync.Mutex
templateIDs map[string]struct{}
flagged bool
}
// New creates a new honeypot detector.
func New(threshold int) *Detector {
if threshold <= 0 {
threshold = 1
}
return &Detector{
threshold: threshold,
}
}
// Threshold returns the distinct template count required to flag a host.
func (d *Detector) Threshold() int {
if d == nil {
return 0
}
return d.threshold
}
// RecordMatch records a match for the given host and templateID.
//
// It returns true only when the host has just become flagged (i.e. crossed the threshold).
func (d *Detector) RecordMatch(host, templateID string) bool {
if d == nil {
return false
}
normalizedHost := normalizeHostKey(host)
if normalizedHost == "" || templateID == "" {
return false
}
stateAny, _ := d.hosts.LoadOrStore(normalizedHost, &hostState{
templateIDs: make(map[string]struct{}),
})
state := stateAny.(*hostState)
state.mu.Lock()
defer state.mu.Unlock()
if state.flagged {
return false
}
if _, ok := state.templateIDs[templateID]; ok {
return false
}
state.templateIDs[templateID] = struct{}{}
if len(state.templateIDs) >= d.threshold {
state.flagged = true
state.templateIDs = nil
return true
}
return false
}
// IsFlagged returns whether the given host is flagged.
func (d *Detector) IsFlagged(host string) bool {
if d == nil {
return false
}
normalizedHost := normalizeHostKey(host)
if normalizedHost == "" {
return false
}
stateAny, ok := d.hosts.Load(normalizedHost)
if !ok {
return false
}
state := stateAny.(*hostState)
state.mu.Lock()
defer state.mu.Unlock()
return state.flagged
}
// Summary returns a short string with the total number of flagged hosts.
func (d *Detector) Summary() string {
if d == nil {
return "honeypot-detected hosts: 0"
}
var flagged int
d.hosts.Range(func(_, v any) bool {
state := v.(*hostState)
state.mu.Lock()
if state.flagged {
flagged++
}
state.mu.Unlock()
return true
})
return fmt.Sprintf("honeypot-detected hosts: %d", flagged)
}
// NormalizeHostKey normalizes host strings so different input formats map to the same key.
func NormalizeHostKey(input string) string {
return normalizeHostKey(input)
}
func normalizeHostKey(input string) string {
s := strings.TrimSpace(input)
if s == "" {
return ""
}
// Strip trailing slashes early.
s = strings.TrimRight(s, "/")
// If an absolute URL is present, parse it to reliably extract host and optional port.
if strings.Contains(s, "://") {
u, err := url.Parse(s)
if err == nil && u != nil {
host := u.Hostname()
port := u.Port()
if host == "" {
return ""
}
host = normalizeHostWithoutPort(host)
if port != "" {
return net.JoinHostPort(host, port)
}
return host
}
// fall through if parsing fails
}
// Remove any path suffix (we only care about the authority).
if idx := strings.IndexByte(s, '/'); idx >= 0 {
s = s[:idx]
}
// If it looks like host:port (including bracketed IPv6), try SplitHostPort first.
if host, port, err := net.SplitHostPort(s); err == nil {
host = normalizeHostWithoutPort(host)
if port == "" {
return host
}
return net.JoinHostPort(host, port)
}
// Handle bracketed IPv6 without port: [2001:db8::1]
if strings.HasPrefix(s, "[") && strings.HasSuffix(s, "]") {
host := strings.TrimSuffix(strings.TrimPrefix(s, "["), "]")
return normalizeHostWithoutPort(host)
}
// Handle bare IPv6 or host without port.
return normalizeHostWithoutPort(s)
}
func normalizeHostWithoutPort(host string) string {
h := strings.TrimSpace(host)
if h == "" {
return ""
}
h = strings.TrimPrefix(h, "[")
h = strings.TrimSuffix(h, "]")
h = strings.ToLower(h)
if ip := net.ParseIP(h); ip != nil {
return ip.String()
}
return h
}
@@ -0,0 +1,108 @@
package honeypotdetector
import (
"fmt"
"sync"
"sync/atomic"
"testing"
"github.com/stretchr/testify/require"
)
func TestThresholdBoundary(t *testing.T) {
t.Parallel()
d := New(2)
host := "example.com"
require.False(t, d.RecordMatch(host, "t1"), "flagging should not happen at N-1 matches")
require.True(t, d.RecordMatch(host, "t2"), "flagging should happen at exactly N distinct template IDs")
require.True(t, d.IsFlagged(host))
// Once flagged, additional distinct matches should not re-trigger the boundary condition.
require.False(t, d.RecordMatch(host, "t3"))
}
func TestDeduplication(t *testing.T) {
t.Parallel()
d := New(2)
host := "example.com"
require.False(t, d.RecordMatch(host, "t1"))
require.False(t, d.RecordMatch(host, "t1"), "same templateID on same host must count once")
require.True(t, d.RecordMatch(host, "t2"))
require.True(t, d.IsFlagged(host))
}
func TestHostIsolation(t *testing.T) {
t.Parallel()
d := New(2)
require.False(t, d.RecordMatch("example-a.com", "t1"))
require.False(t, d.RecordMatch("example-b.com", "t1"))
require.True(t, d.RecordMatch("example-a.com", "t2"), "host A should be flagged independently")
require.True(t, d.IsFlagged("example-a.com"))
require.False(t, d.IsFlagged("example-b.com"))
}
func TestConcurrentAccess(t *testing.T) {
t.Parallel()
const (
threshold = 10
goroutines = 100
)
d := New(threshold)
host := "example.com"
var justFlaggedCount atomic.Int32
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
i := i
go func() {
defer wg.Done()
if d.RecordMatch(host, fmt.Sprintf("t-%d", i)) {
justFlaggedCount.Add(1)
}
}()
}
wg.Wait()
require.True(t, d.IsFlagged(host))
require.Equal(t, int32(1), justFlaggedCount.Load(), "exactly one goroutine should trigger the boundary")
}
func TestHostNormalization(t *testing.T) {
t.Parallel()
d := New(2)
// Scheme + trailing slash must be stripped so these map to the same normalized host.
require.False(t, d.RecordMatch("https://example.com/", "t1"))
require.True(t, d.RecordMatch("http://example.com", "t2"))
require.True(t, d.IsFlagged("example.com"))
// Explicit port should keep it distinct (example.com and example.com:443 are different keys).
d2 := New(1)
require.True(t, d2.RecordMatch("example.com:443", "t1"))
require.True(t, d2.IsFlagged("example.com:443"))
require.False(t, d2.IsFlagged("example.com"), "host without explicit port must not share the same key")
// IPv6 should be canonicalized, and bracketed host:port should keep bracketed form when port is present.
d3 := New(1)
require.True(t, d3.RecordMatch("http://[2001:db8::1]/", "t1"))
require.True(t, d3.IsFlagged("2001:db8::1"))
require.True(t, d3.IsFlagged("[2001:db8::1]"))
d4 := New(1)
require.True(t, d4.RecordMatch("[2001:db8::1]:443", "t1"))
require.True(t, d4.IsFlagged("[2001:db8::1]:443"))
require.False(t, d4.IsFlagged("2001:db8::1"), "IPv6 with explicit port must not share the same key as IPv6 without port")
}
+10
View File
@@ -258,6 +258,13 @@ type Options struct {
Stdin bool
// StopAtFirstMatch stops processing template at first full match (this may break chained requests)
StopAtFirstMatch bool
// HoneypotDetection enables detection of potential honeypots based on match concentration.
HoneypotDetection bool
// HoneypotThreshold is the number of distinct template IDs required to flag a host as a potential honeypot.
HoneypotThreshold int
// SuppressHoneypotResults suppresses output writing for flagged honeypot hosts.
SuppressHoneypotResults bool
// Stream the input without sorting
Stream bool
// NoMeta disables display of metadata for the matches
@@ -588,6 +595,9 @@ func (options *Options) Copy() *Options {
HangMonitor: options.HangMonitor,
Stdin: options.Stdin,
StopAtFirstMatch: options.StopAtFirstMatch,
HoneypotDetection: options.HoneypotDetection,
HoneypotThreshold: options.HoneypotThreshold,
SuppressHoneypotResults: options.SuppressHoneypotResults,
Stream: options.Stream,
NoMeta: options.NoMeta,
Timestamp: options.Timestamp,