admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2463 from projectdiscovery/2014-ip-aware-duplicate-filter

Browse Source 
adding ip aware dedup
This commit is contained in:
Dogan Can Bakir
2026-03-25 14:11:38 +03:00
committed by GitHub
parent 9eebed0deb 7d765f4372
commit 2cc533074e
2 changed files with 154 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runner/runner.go
+13 -11
View File
@@ -95,7 +95,7 @@ type Runner struct {
browser *Browser
ditClassifier *dit.Classifier
pHashClusters []pHashCluster
simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection
simHashes gcache.Cache[uint64, []string]
httpApiEndpoint *Server
authProvider authprovider.AuthProvider
interruptCh chan struct{}
@@ -430,7 +430,7 @@ func New(options *Options) (*Runner, error) {
runner.HostErrorsCache = gc
}
runner.simHashes = gcache.New[uint64, struct{}](1000).ARC().Build()
runner.simHashes = gcache.New[uint64, []string](1000).ARC().Build()
if options.JSONOutput || options.CSVOutput || len(options.OutputFilterPageType) > 0 {
ditClassifier, err := dit.New()
if err != nil {
@@ -639,19 +639,21 @@ func (r *Runner) seen(k string) bool {
func (r *Runner) duplicate(result *Result) bool {
respSimHash := simhash.Simhash(simhash.NewWordFeatureSet(converstionutil.Bytes(result.Raw)))
if r.simHashes.Has(respSimHash) {
gologger.Debug().Msgf("Skipping duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
return true
}
ip := result.HostIP
for simHash := range r.simHashes.GetALL(false) {
// lower threshold for increased precision
if simhash.Compare(simHash, respSimHash) <= 3 {
gologger.Debug().Msgf("Skipping near-duplicate response with simhash %d for URL %s\n", respSimHash, result.URL)
for storedHash, storedIPs := range r.simHashes.GetALL(false) {
if simhash.Compare(storedHash, respSimHash) > 3 {
continue
}
if ip == "" || sliceutil.Contains(storedIPs, ip) {
gologger.Debug().Msgf("Skipping duplicate response (simhash %d, ip %s) for URL %s\n", respSimHash, ip, result.URL)
return true
}
_ = r.simHashes.Set(storedHash, append(storedIPs, ip))
return false
}
_ = r.simHashes.Set(respSimHash, struct{}{})
_ = r.simHashes.Set(respSimHash, []string{ip})
return false
}
runner/runner_test.go
+141
View File
@@ -386,6 +386,147 @@ func TestRunner_testAndSet_concurrent(t *testing.T) {
require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key")
}
func TestRunner_duplicate(t *testing.T) {
const (
pageA = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here</body></html>"
pageB = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Dashboard</title></head><body>Completely different application running on this server</body></html>"
)
t.Run("same content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://b.example.com"}
require.False(t, r.duplicate(first), "first result should not be duplicate")
require.True(t, r.duplicate(second), "same content + same IP should be duplicate")
})
t.Run("same content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "same content but different IP should NOT be duplicate")
})
t.Run("different content same IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageB, HostIP: "1.1.1.1", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different content on same IP should NOT be duplicate")
})
t.Run("different content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageB, HostIP: "2.2.2.2", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different content + different IP should NOT be duplicate")
})
t.Run("third subdomain same content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
third := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://c.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different IP should be kept")
require.True(t, r.duplicate(third), "same content + same IP as first should be duplicate")
})
t.Run("near-duplicate content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
nearDup := &Result{
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
HostIP: "1.1.1.1",
URL: "https://b.example.com",
}
require.False(t, r.duplicate(first))
require.True(t, r.duplicate(nearDup), "near-duplicate content from same IP should be duplicate")
})
t.Run("near-duplicate content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
nearDup := &Result{
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
HostIP: "3.3.3.3",
URL: "https://b.example.com",
}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(nearDup), "near-duplicate content from different IP should NOT be duplicate")
})
t.Run("empty IP falls back to content-only dedup", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.True(t, r.duplicate(second), "empty IP should fall back to content-only dedup")
})
t.Run("many subdomains same default page same IP", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
kept := 0
for i := 0; i < 50; i++ {
res := &Result{
Raw: pageA,
HostIP: "10.0.0.1",
URL: fmt.Sprintf("https://sub%d.example.com", i),
}
if !r.duplicate(res) {
kept++
}
}
require.Equal(t, 1, kept, "50 subdomains with identical content on same IP should keep exactly 1")
})
t.Run("many subdomains same default page different IPs", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
kept := 0
for i := 0; i < 50; i++ {
res := &Result{
Raw: pageA,
HostIP: fmt.Sprintf("10.0.0.%d", i+1),
URL: fmt.Sprintf("https://sub%d.example.com", i),
}
if !r.duplicate(res) {
kept++
}
}
require.Equal(t, 50, kept, "50 subdomains with identical content but different IPs should keep all 50")
})
}
func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) {
runner := &Runner{}