Files
Doğan Can Bakır 98e6af00f0 Merge remote-tracking branch 'origin/dev' into pr-2462
# Conflicts:
#	runner/runner_test.go
2026-05-06 16:22:17 +03:00

771 lines
22 KiB
Go

package runner
import (
"fmt"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/pkg/errors"
_ "github.com/projectdiscovery/fdmax/autofdmax"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/mapcidr/asn"
stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/stretchr/testify/require"
)
func TestRunner_resumeAfterInterrupt(t *testing.T) {
domains := []string{"a.com", "b.com", "c.com", "d.com", "e.com", "f.com", "g.com", "h.com", "i.com", "j.com"}
interruptAfter := 4
// --- Full scan (reference): process all domains without interrupt ---
rFull, err := New(&Options{})
require.Nil(t, err, "could not create httpx runner")
rFull.options.resumeCfg = &ResumeCfg{}
var fullOutput []string
for _, d := range domains {
rFull.options.resumeCfg.current = d
rFull.options.resumeCfg.currentIndex++
fullOutput = append(fullOutput, d)
}
// --- Interrupted scan: process items, interrupt after interruptAfter ---
rInt, err := New(&Options{})
require.Nil(t, err, "could not create httpx runner")
rInt.options.resumeCfg = &ResumeCfg{}
var interruptedOutput []string
for _, d := range domains {
// same check as processItem: bail out if interrupted
select {
case <-rInt.interruptCh:
continue
default:
}
rInt.options.resumeCfg.current = d
rInt.options.resumeCfg.currentIndex++
interruptedOutput = append(interruptedOutput, d)
if len(interruptedOutput) == interruptAfter {
rInt.Interrupt()
}
}
// simulate SaveResumeConfig: save the index after interrupt
savedIndex := rInt.options.resumeCfg.currentIndex
// the saved index must equal exactly the number of items that were processed
require.Equal(t, interruptAfter, savedIndex, "resume index should equal number of completed items")
// every domain before the index must be in the interrupted output
require.Equal(t, domains[:interruptAfter], interruptedOutput, "interrupted output should contain exactly the first N domains")
// --- Resumed scan: load saved index, skip already-processed items ---
rRes, err := New(&Options{})
require.Nil(t, err, "could not create httpx runner")
rRes.options.resumeCfg = &ResumeCfg{Index: savedIndex}
var resumedOutput []string
for _, d := range domains {
// same resume-skip logic as processItem
rRes.options.resumeCfg.current = d
rRes.options.resumeCfg.currentIndex++
if rRes.options.resumeCfg.currentIndex <= rRes.options.resumeCfg.Index {
continue
}
resumedOutput = append(resumedOutput, d)
}
// every domain after the index must be in the resumed output
require.Equal(t, domains[interruptAfter:], resumedOutput, "resumed output should contain exactly the remaining domains")
// union of interrupted + resumed must equal the full scan
combined := append(interruptedOutput, resumedOutput...)
require.Equal(t, fullOutput, combined, "interrupted + resumed should equal full scan")
}
func TestRunner_domain_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := []string{"example.com", "*.example.com", "example.com,one.one.one.one"}
expected := []httpx.Target{{
Host: "example.com",
}, {
Host: "example.com",
}, {
Host: "one.one.one.one",
CustomHost: "example.com",
}}
got := []httpx.Target{}
for _, inp := range input {
for target := range r.targets(r.hp, inp) {
got = append(got, target)
}
}
require.ElementsMatch(t, expected, got, "could not expected output")
}
func TestRunner_probeall_targets(t *testing.T) {
options := &Options{
ProbeAllIPS: true,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "one.one.one.one"
expected := []httpx.Target{{
Host: "one.one.one.one",
CustomIP: "2606:4700:4700::1111",
},
{
Host: "one.one.one.one",
CustomIP: "2606:4700:4700::1001",
},
{
Host: "one.one.one.one",
CustomIP: "1.0.0.1",
},
{
Host: "one.one.one.one",
CustomIP: "1.1.1.1",
}}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not expected output")
}
func TestRunner_probeall_targets_with_port(t *testing.T) {
options := &Options{
ProbeAllIPS: true,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
inputWithPort := "http://one.one.one.one:8080"
inputWithoutPort := "one.one.one.one"
gotWithPort := []httpx.Target{}
for target := range r.targets(r.hp, inputWithPort) {
gotWithPort = append(gotWithPort, target)
}
gotWithoutPort := []httpx.Target{}
for target := range r.targets(r.hp, inputWithoutPort) {
gotWithoutPort = append(gotWithoutPort, target)
}
require.True(t, len(gotWithPort) > 0, "probe-all-ips with port should return at least one target")
require.True(t, len(gotWithoutPort) > 0, "probe-all-ips without port should return at least one target")
require.Equal(t, len(gotWithPort), len(gotWithoutPort), "probe-all-ips should return same number of IPs with or without port")
for _, target := range gotWithPort {
require.Equal(t, inputWithPort, target.Host, "Host should be preserved with port")
require.NotEmpty(t, target.CustomIP, "CustomIP should be populated")
}
}
func TestRunner_cidr_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "173.0.84.0/30"
expected := []httpx.Target{
{
Host: "173.0.84.0",
}, {
Host: "173.0.84.1",
},
{
Host: "173.0.84.2",
},
{
Host: "173.0.84.3",
}}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not expected output")
}
func TestRunner_asn_targets(t *testing.T) {
if os.Getenv("PDCP_API_KEY") == "" {
return
}
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "AS14421"
expected := []httpx.Target{}
expectedOutputFile := "tests/AS14421.txt"
// read the expected IPs from the file
fileContent, err := os.ReadFile(expectedOutputFile)
require.Nil(t, err, "could not read the expectedOutputFile file")
ips := strings.Split(strings.ReplaceAll(string(fileContent), "\r\n", "\n"), "\n")
for _, ip := range ips {
expected = append(expected, httpx.Target{Host: ip})
}
if _, err := asn.GetIPAddressesAsStream(input); err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
t.Skip("skipping asn test due to missing/invalid api key")
return
}
got := []httpx.Target{}
for target := range r.targets(r.hp, input) {
got = append(got, target)
}
require.ElementsMatch(t, expected, got, "could not get expected output")
}
func TestRunner_countTargetFromRawTarget(t *testing.T) {
options := &Options{
SkipDedupe: false,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := "example.com"
expected := 1
got, err := r.countTargetFromRawTarget(input)
require.Nil(t, err, "could not count targets")
require.Equal(t, expected, got, "got wrong output")
input = "example.com"
expected = 0
err = r.hm.Set(input, nil)
require.Nil(t, err, "could not set value to hm")
got, err = r.countTargetFromRawTarget(input)
require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error")
require.Equal(t, expected, got, "got wrong output")
input = "173.0.84.0/24"
expected = 256
got, err = r.countTargetFromRawTarget(input)
require.Nil(t, err, "could not count targets")
require.Equal(t, expected, got, "got wrong output")
input = ""
expected = 0
got, err = r.countTargetFromRawTarget(input)
require.Nil(t, err, "could not count targets")
require.Equal(t, expected, got, "got wrong output")
if os.Getenv("PDCP_API_KEY") != "" {
input = "AS14421"
expected = 256
got, err = r.countTargetFromRawTarget(input)
if err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
t.Skip("skipping asn test due to missing/invalid api key")
return
}
require.Nil(t, err, "could not count targets")
require.Equal(t, expected, got, "got wrong output")
}
}
func TestRunner_urlWithComma_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
input := []string{"http://scanme.sh?a=1,2"}
expected := []httpx.Target{{
Host: "http://scanme.sh?a=1,2",
}}
got := []httpx.Target{}
for _, inp := range input {
for target := range r.targets(r.hp, inp) {
got = append(got, target)
}
}
require.ElementsMatch(t, expected, got, "could not expected output")
}
func TestRunner_CSVRow(t *testing.T) {
// Create a result with fields that would be vulnerable to CSV injection
result := Result{
URL: `=HYPERLINK('https://evil.com','click me')`,
Title: `+CMD('calc')`,
ContentType: `-SUM(1+1)`,
WebServer: `@MACRO=Virus()`,
StatusCode: 200,
Timestamp: time.Now(),
}
// Call CSVRow to get the sanitized output
csvOutput := result.CSVRow(nil)
// Check that vulnerable fields are properly sanitized with a prefix quote
tests := []struct {
fieldName string
original string
expected string
}{
{"URL", result.URL, fmt.Sprintf("'%s", result.URL)},
{"Title", result.Title, fmt.Sprintf("'%s", result.Title)},
{"ContentType", result.ContentType, fmt.Sprintf("'%s", result.ContentType)},
{"WebServer", result.WebServer, fmt.Sprintf("'%s", result.WebServer)},
}
for _, tc := range tests {
if !strings.Contains(csvOutput, tc.expected) {
t.Errorf("CSV sanitization failed for %s field: expected %q but sanitized value not found in output: %s",
tc.fieldName, tc.expected, csvOutput)
}
}
// Also check that normal fields remain unsanitized
if strings.Contains(csvOutput, "'200") {
t.Error("CSV sanitization incorrectly modified non-vulnerable field")
}
}
func TestRunner_testAndSet(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err, "could not create httpx runner")
t.Run("first insert returns true", func(t *testing.T) {
require.True(t, r.testAndSet("example.com"))
})
t.Run("duplicate returns false", func(t *testing.T) {
require.False(t, r.testAndSet("example.com"))
})
t.Run("different key returns true", func(t *testing.T) {
require.True(t, r.testAndSet("other.com"))
})
t.Run("empty string returns false", func(t *testing.T) {
require.False(t, r.testAndSet(""))
})
t.Run("whitespace-only returns false", func(t *testing.T) {
require.False(t, r.testAndSet(" "))
})
t.Run("trimmed duplicate returns false", func(t *testing.T) {
require.False(t, r.testAndSet(" example.com "))
})
}
func TestRunner_testAndSet_concurrent(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err, "could not create httpx runner")
const goroutines = 100
key := "race-target.com"
wins := make([]bool, goroutines)
var wg sync.WaitGroup
wg.Add(goroutines)
start := make(chan struct{})
for i := 0; i < goroutines; i++ {
go func(idx int) {
defer wg.Done()
<-start
wins[idx] = r.testAndSet(key)
}(i)
}
close(start)
wg.Wait()
winCount := 0
for _, w := range wins {
if w {
winCount++
}
}
require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key")
}
func TestOptions_hasMatcherOrFilter(t *testing.T) {
tests := []struct {
name string
options Options
expected bool
}{
{
name: "no matchers or filters",
options: Options{},
expected: false,
},
{
name: "match status code",
options: Options{OutputMatchStatusCode: "200"},
expected: true,
},
{
name: "filter status code",
options: Options{OutputFilterStatusCode: "403,401"},
expected: true,
},
{
name: "match string",
options: Options{OutputMatchString: []string{"admin"}},
expected: true,
},
{
name: "filter string",
options: Options{OutputFilterString: []string{"error"}},
expected: true,
},
{
name: "match content length",
options: Options{OutputMatchContentLength: "100"},
expected: true,
},
{
name: "filter content length",
options: Options{OutputFilterContentLength: "0"},
expected: true,
},
{
name: "match regex",
options: Options{OutputMatchRegex: []string{"admin.*panel"}},
expected: true,
},
{
name: "filter regex",
options: Options{OutputFilterRegex: []string{"error"}},
expected: true,
},
{
name: "match lines count",
options: Options{OutputMatchLinesCount: "50"},
expected: true,
},
{
name: "filter lines count",
options: Options{OutputFilterLinesCount: "0"},
expected: true,
},
{
name: "match words count",
options: Options{OutputMatchWordsCount: "100"},
expected: true,
},
{
name: "filter words count",
options: Options{OutputFilterWordsCount: "0"},
expected: true,
},
{
name: "match favicon",
options: Options{OutputMatchFavicon: []string{"1494302000"}},
expected: true,
},
{
name: "filter favicon",
options: Options{OutputFilterFavicon: []string{"1494302000"}},
expected: true,
},
{
name: "match cdn",
options: Options{OutputMatchCdn: []string{"cloudflare"}},
expected: true,
},
{
name: "filter cdn",
options: Options{OutputFilterCdn: []string{"cloudflare"}},
expected: true,
},
{
name: "match condition",
options: Options{OutputMatchCondition: "status_code == 200"},
expected: true,
},
{
name: "filter condition",
options: Options{OutputFilterCondition: "status_code == 403"},
expected: true,
},
{
name: "match response time",
options: Options{OutputMatchResponseTime: "< 1"},
expected: true,
},
{
name: "filter response time",
options: Options{OutputFilterResponseTime: "> 5"},
expected: true,
},
{
name: "filter page type",
options: Options{OutputFilterPageType: []string{"error"}},
expected: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
opts := tc.options
err := opts.ValidateOptions()
require.Nil(t, err)
require.Equal(t, tc.expected, opts.HasMatcherOrFilter(),
"HasMatcherOrFilter() should be %v for %s", tc.expected, tc.name)
})
}
}
func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
dir := t.TempDir()
opts := &Options{
StoreResponse: true,
StoreResponseDir: dir,
}
err := opts.ValidateOptions()
require.Nil(t, err)
require.False(t, opts.HasMatcherOrFilter())
}
func TestStoreResponse_withMatcherSetsFlag(t *testing.T) {
dir := t.TempDir()
opts := &Options{
StoreResponse: true,
StoreResponseDir: dir,
OutputMatchStatusCode: "200",
}
err := opts.ValidateOptions()
require.Nil(t, err)
require.True(t, opts.HasMatcherOrFilter())
}
func TestRunner_duplicate(t *testing.T) {
const (
pageA = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here</body></html>"
pageB = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Dashboard</title></head><body>Completely different application running on this server</body></html>"
)
t.Run("same content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://b.example.com"}
require.False(t, r.duplicate(first), "first result should not be duplicate")
require.True(t, r.duplicate(second), "same content + same IP should be duplicate")
})
t.Run("same content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "same content but different IP should NOT be duplicate")
})
t.Run("different content same IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageB, HostIP: "1.1.1.1", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different content on same IP should NOT be duplicate")
})
t.Run("different content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageB, HostIP: "2.2.2.2", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different content + different IP should NOT be duplicate")
})
t.Run("third subdomain same content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
third := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://c.example.com"}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(second), "different IP should be kept")
require.True(t, r.duplicate(third), "same content + same IP as first should be duplicate")
})
t.Run("near-duplicate content same IP is duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
nearDup := &Result{
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
HostIP: "1.1.1.1",
URL: "https://b.example.com",
}
require.False(t, r.duplicate(first))
require.True(t, r.duplicate(nearDup), "near-duplicate content from same IP should be duplicate")
})
t.Run("near-duplicate content different IP is NOT duplicate", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
nearDup := &Result{
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
HostIP: "3.3.3.3",
URL: "https://b.example.com",
}
require.False(t, r.duplicate(first))
require.False(t, r.duplicate(nearDup), "near-duplicate content from different IP should NOT be duplicate")
})
t.Run("empty IP falls back to content-only dedup", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
first := &Result{Raw: pageA, HostIP: "", URL: "https://a.example.com"}
second := &Result{Raw: pageA, HostIP: "", URL: "https://b.example.com"}
require.False(t, r.duplicate(first))
require.True(t, r.duplicate(second), "empty IP should fall back to content-only dedup")
})
t.Run("many subdomains same default page same IP", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
kept := 0
for i := 0; i < 50; i++ {
res := &Result{
Raw: pageA,
HostIP: "10.0.0.1",
URL: fmt.Sprintf("https://sub%d.example.com", i),
}
if !r.duplicate(res) {
kept++
}
}
require.Equal(t, 1, kept, "50 subdomains with identical content on same IP should keep exactly 1")
})
t.Run("many subdomains same default page different IPs", func(t *testing.T) {
r, err := New(&Options{})
require.Nil(t, err)
kept := 0
for i := 0; i < 50; i++ {
res := &Result{
Raw: pageA,
HostIP: fmt.Sprintf("10.0.0.%d", i+1),
URL: fmt.Sprintf("https://sub%d.example.com", i),
}
if !r.duplicate(res) {
kept++
}
}
require.Equal(t, 50, kept, "50 subdomains with identical content but different IPs should keep all 50")
})
}
func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) {
runner := &Runner{}
tests := []struct {
name string
allow []string
deny []string
testCases []struct {
ip string
expected bool
reason string
}
}{
{
name: "Allow flag blocks IPs outside allowed range",
allow: []string{"192.168.1.0/24"},
deny: nil,
testCases: []struct {
ip string
expected bool
reason string
}{
{"8.8.8.8", false, "IP outside allowed range should be blocked"},
{"192.168.1.10", true, "IP inside allowed range should be allowed"},
},
},
{
name: "Deny flag blocks IPs in denied range",
allow: nil,
deny: []string{"127.0.0.0/8"},
testCases: []struct {
ip string
expected bool
reason string
}{
{"127.0.0.1", false, "IP in denied range should be blocked"},
{"8.8.8.8", true, "IP outside denied range should be allowed"},
},
},
{
name: "Combined Allow and Deny flags",
allow: []string{"192.168.0.0/16"},
deny: []string{"192.168.1.0/24"},
testCases: []struct {
ip string
expected bool
reason string
}{
{"10.0.0.1", false, "IP outside allowed range should be blocked"},
{"192.168.1.100", false, "IP in denied range should be blocked even if in allowed range"},
{"192.168.2.50", true, "IP in allowed range but not in denied range should be allowed"},
},
},
{
name: "Multiple Allow and Deny ranges",
allow: []string{"10.0.0.0/8", "172.16.0.0/12"},
deny: []string{"10.1.0.0/16", "172.20.0.0/16"},
testCases: []struct {
ip string
expected bool
reason string
}{
{"10.0.1.1", true, "10.0.1.1 should be allowed (in allow range, not in deny)"},
{"10.1.1.1", false, "10.1.1.1 should be blocked (in deny range)"},
{"172.16.1.1", true, "172.16.1.1 should be allowed (in allow range, not in deny)"},
{"172.20.1.1", false, "172.20.1.1 should be blocked (in deny range)"},
{"192.168.1.1", false, "192.168.1.1 should be blocked (not in any allow range)"},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
options := &Options{
Allow: tc.allow,
Deny: tc.deny,
}
np, err := runner.createNetworkpolicyInstance(options)
require.Nil(t, err, "could not create networkpolicy instance")
require.NotNil(t, np, "networkpolicy instance should not be nil")
for _, testCase := range tc.testCases {
allowed := np.Validate(testCase.ip)
require.Equal(t, testCase.expected, allowed, testCase.reason)
}
})
}
}