mirror of
https://github.com/projectdiscovery/httpx
synced 2026-06-08 16:50:17 +00:00
98e6af00f0
# Conflicts: # runner/runner_test.go
771 lines
22 KiB
Go
771 lines
22 KiB
Go
package runner
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
_ "github.com/projectdiscovery/fdmax/autofdmax"
|
|
"github.com/projectdiscovery/httpx/common/httpx"
|
|
"github.com/projectdiscovery/mapcidr/asn"
|
|
stringsutil "github.com/projectdiscovery/utils/strings"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestRunner_resumeAfterInterrupt(t *testing.T) {
|
|
domains := []string{"a.com", "b.com", "c.com", "d.com", "e.com", "f.com", "g.com", "h.com", "i.com", "j.com"}
|
|
interruptAfter := 4
|
|
|
|
// --- Full scan (reference): process all domains without interrupt ---
|
|
rFull, err := New(&Options{})
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
rFull.options.resumeCfg = &ResumeCfg{}
|
|
var fullOutput []string
|
|
for _, d := range domains {
|
|
rFull.options.resumeCfg.current = d
|
|
rFull.options.resumeCfg.currentIndex++
|
|
fullOutput = append(fullOutput, d)
|
|
}
|
|
|
|
// --- Interrupted scan: process items, interrupt after interruptAfter ---
|
|
rInt, err := New(&Options{})
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
rInt.options.resumeCfg = &ResumeCfg{}
|
|
var interruptedOutput []string
|
|
for _, d := range domains {
|
|
// same check as processItem: bail out if interrupted
|
|
select {
|
|
case <-rInt.interruptCh:
|
|
continue
|
|
default:
|
|
}
|
|
|
|
rInt.options.resumeCfg.current = d
|
|
rInt.options.resumeCfg.currentIndex++
|
|
interruptedOutput = append(interruptedOutput, d)
|
|
|
|
if len(interruptedOutput) == interruptAfter {
|
|
rInt.Interrupt()
|
|
}
|
|
}
|
|
|
|
// simulate SaveResumeConfig: save the index after interrupt
|
|
savedIndex := rInt.options.resumeCfg.currentIndex
|
|
|
|
// the saved index must equal exactly the number of items that were processed
|
|
require.Equal(t, interruptAfter, savedIndex, "resume index should equal number of completed items")
|
|
// every domain before the index must be in the interrupted output
|
|
require.Equal(t, domains[:interruptAfter], interruptedOutput, "interrupted output should contain exactly the first N domains")
|
|
|
|
// --- Resumed scan: load saved index, skip already-processed items ---
|
|
rRes, err := New(&Options{})
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
rRes.options.resumeCfg = &ResumeCfg{Index: savedIndex}
|
|
var resumedOutput []string
|
|
for _, d := range domains {
|
|
// same resume-skip logic as processItem
|
|
rRes.options.resumeCfg.current = d
|
|
rRes.options.resumeCfg.currentIndex++
|
|
if rRes.options.resumeCfg.currentIndex <= rRes.options.resumeCfg.Index {
|
|
continue
|
|
}
|
|
resumedOutput = append(resumedOutput, d)
|
|
}
|
|
|
|
// every domain after the index must be in the resumed output
|
|
require.Equal(t, domains[interruptAfter:], resumedOutput, "resumed output should contain exactly the remaining domains")
|
|
|
|
// union of interrupted + resumed must equal the full scan
|
|
combined := append(interruptedOutput, resumedOutput...)
|
|
require.Equal(t, fullOutput, combined, "interrupted + resumed should equal full scan")
|
|
}
|
|
|
|
func TestRunner_domain_targets(t *testing.T) {
|
|
options := &Options{}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
input := []string{"example.com", "*.example.com", "example.com,one.one.one.one"}
|
|
expected := []httpx.Target{{
|
|
Host: "example.com",
|
|
}, {
|
|
Host: "example.com",
|
|
}, {
|
|
Host: "one.one.one.one",
|
|
CustomHost: "example.com",
|
|
}}
|
|
got := []httpx.Target{}
|
|
for _, inp := range input {
|
|
for target := range r.targets(r.hp, inp) {
|
|
got = append(got, target)
|
|
}
|
|
}
|
|
require.ElementsMatch(t, expected, got, "could not expected output")
|
|
}
|
|
|
|
func TestRunner_probeall_targets(t *testing.T) {
|
|
options := &Options{
|
|
ProbeAllIPS: true,
|
|
}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
input := "one.one.one.one"
|
|
expected := []httpx.Target{{
|
|
Host: "one.one.one.one",
|
|
CustomIP: "2606:4700:4700::1111",
|
|
},
|
|
{
|
|
Host: "one.one.one.one",
|
|
CustomIP: "2606:4700:4700::1001",
|
|
},
|
|
{
|
|
Host: "one.one.one.one",
|
|
CustomIP: "1.0.0.1",
|
|
},
|
|
{
|
|
Host: "one.one.one.one",
|
|
CustomIP: "1.1.1.1",
|
|
}}
|
|
got := []httpx.Target{}
|
|
for target := range r.targets(r.hp, input) {
|
|
got = append(got, target)
|
|
}
|
|
|
|
require.ElementsMatch(t, expected, got, "could not expected output")
|
|
}
|
|
|
|
func TestRunner_probeall_targets_with_port(t *testing.T) {
|
|
options := &Options{
|
|
ProbeAllIPS: true,
|
|
}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
|
|
inputWithPort := "http://one.one.one.one:8080"
|
|
inputWithoutPort := "one.one.one.one"
|
|
|
|
gotWithPort := []httpx.Target{}
|
|
for target := range r.targets(r.hp, inputWithPort) {
|
|
gotWithPort = append(gotWithPort, target)
|
|
}
|
|
|
|
gotWithoutPort := []httpx.Target{}
|
|
for target := range r.targets(r.hp, inputWithoutPort) {
|
|
gotWithoutPort = append(gotWithoutPort, target)
|
|
}
|
|
|
|
require.True(t, len(gotWithPort) > 0, "probe-all-ips with port should return at least one target")
|
|
require.True(t, len(gotWithoutPort) > 0, "probe-all-ips without port should return at least one target")
|
|
require.Equal(t, len(gotWithPort), len(gotWithoutPort), "probe-all-ips should return same number of IPs with or without port")
|
|
|
|
for _, target := range gotWithPort {
|
|
require.Equal(t, inputWithPort, target.Host, "Host should be preserved with port")
|
|
require.NotEmpty(t, target.CustomIP, "CustomIP should be populated")
|
|
}
|
|
}
|
|
|
|
func TestRunner_cidr_targets(t *testing.T) {
|
|
options := &Options{}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
input := "173.0.84.0/30"
|
|
expected := []httpx.Target{
|
|
{
|
|
Host: "173.0.84.0",
|
|
}, {
|
|
Host: "173.0.84.1",
|
|
},
|
|
{
|
|
Host: "173.0.84.2",
|
|
},
|
|
{
|
|
Host: "173.0.84.3",
|
|
}}
|
|
got := []httpx.Target{}
|
|
for target := range r.targets(r.hp, input) {
|
|
got = append(got, target)
|
|
}
|
|
|
|
require.ElementsMatch(t, expected, got, "could not expected output")
|
|
}
|
|
|
|
func TestRunner_asn_targets(t *testing.T) {
|
|
if os.Getenv("PDCP_API_KEY") == "" {
|
|
return
|
|
}
|
|
|
|
options := &Options{}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
input := "AS14421"
|
|
expected := []httpx.Target{}
|
|
expectedOutputFile := "tests/AS14421.txt"
|
|
// read the expected IPs from the file
|
|
fileContent, err := os.ReadFile(expectedOutputFile)
|
|
require.Nil(t, err, "could not read the expectedOutputFile file")
|
|
ips := strings.Split(strings.ReplaceAll(string(fileContent), "\r\n", "\n"), "\n")
|
|
for _, ip := range ips {
|
|
expected = append(expected, httpx.Target{Host: ip})
|
|
}
|
|
|
|
if _, err := asn.GetIPAddressesAsStream(input); err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
|
|
t.Skip("skipping asn test due to missing/invalid api key")
|
|
return
|
|
}
|
|
|
|
got := []httpx.Target{}
|
|
for target := range r.targets(r.hp, input) {
|
|
got = append(got, target)
|
|
}
|
|
require.ElementsMatch(t, expected, got, "could not get expected output")
|
|
}
|
|
|
|
func TestRunner_countTargetFromRawTarget(t *testing.T) {
|
|
options := &Options{
|
|
SkipDedupe: false,
|
|
}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
|
|
input := "example.com"
|
|
expected := 1
|
|
got, err := r.countTargetFromRawTarget(input)
|
|
require.Nil(t, err, "could not count targets")
|
|
require.Equal(t, expected, got, "got wrong output")
|
|
|
|
input = "example.com"
|
|
expected = 0
|
|
err = r.hm.Set(input, nil)
|
|
require.Nil(t, err, "could not set value to hm")
|
|
got, err = r.countTargetFromRawTarget(input)
|
|
require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error")
|
|
require.Equal(t, expected, got, "got wrong output")
|
|
|
|
input = "173.0.84.0/24"
|
|
expected = 256
|
|
got, err = r.countTargetFromRawTarget(input)
|
|
require.Nil(t, err, "could not count targets")
|
|
require.Equal(t, expected, got, "got wrong output")
|
|
|
|
input = ""
|
|
expected = 0
|
|
got, err = r.countTargetFromRawTarget(input)
|
|
require.Nil(t, err, "could not count targets")
|
|
require.Equal(t, expected, got, "got wrong output")
|
|
|
|
if os.Getenv("PDCP_API_KEY") != "" {
|
|
input = "AS14421"
|
|
expected = 256
|
|
got, err = r.countTargetFromRawTarget(input)
|
|
if err != nil && stringsutil.ContainsAnyI(err.Error(), "unauthorized: 401") {
|
|
t.Skip("skipping asn test due to missing/invalid api key")
|
|
return
|
|
}
|
|
require.Nil(t, err, "could not count targets")
|
|
require.Equal(t, expected, got, "got wrong output")
|
|
}
|
|
}
|
|
|
|
func TestRunner_urlWithComma_targets(t *testing.T) {
|
|
options := &Options{}
|
|
r, err := New(options)
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
input := []string{"http://scanme.sh?a=1,2"}
|
|
expected := []httpx.Target{{
|
|
Host: "http://scanme.sh?a=1,2",
|
|
}}
|
|
got := []httpx.Target{}
|
|
for _, inp := range input {
|
|
for target := range r.targets(r.hp, inp) {
|
|
got = append(got, target)
|
|
}
|
|
}
|
|
require.ElementsMatch(t, expected, got, "could not expected output")
|
|
}
|
|
|
|
func TestRunner_CSVRow(t *testing.T) {
|
|
// Create a result with fields that would be vulnerable to CSV injection
|
|
result := Result{
|
|
URL: `=HYPERLINK('https://evil.com','click me')`,
|
|
Title: `+CMD('calc')`,
|
|
ContentType: `-SUM(1+1)`,
|
|
WebServer: `@MACRO=Virus()`,
|
|
StatusCode: 200,
|
|
Timestamp: time.Now(),
|
|
}
|
|
|
|
// Call CSVRow to get the sanitized output
|
|
csvOutput := result.CSVRow(nil)
|
|
|
|
// Check that vulnerable fields are properly sanitized with a prefix quote
|
|
tests := []struct {
|
|
fieldName string
|
|
original string
|
|
expected string
|
|
}{
|
|
{"URL", result.URL, fmt.Sprintf("'%s", result.URL)},
|
|
{"Title", result.Title, fmt.Sprintf("'%s", result.Title)},
|
|
{"ContentType", result.ContentType, fmt.Sprintf("'%s", result.ContentType)},
|
|
{"WebServer", result.WebServer, fmt.Sprintf("'%s", result.WebServer)},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
if !strings.Contains(csvOutput, tc.expected) {
|
|
t.Errorf("CSV sanitization failed for %s field: expected %q but sanitized value not found in output: %s",
|
|
tc.fieldName, tc.expected, csvOutput)
|
|
}
|
|
}
|
|
|
|
// Also check that normal fields remain unsanitized
|
|
if strings.Contains(csvOutput, "'200") {
|
|
t.Error("CSV sanitization incorrectly modified non-vulnerable field")
|
|
}
|
|
}
|
|
|
|
func TestRunner_testAndSet(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
|
|
t.Run("first insert returns true", func(t *testing.T) {
|
|
require.True(t, r.testAndSet("example.com"))
|
|
})
|
|
|
|
t.Run("duplicate returns false", func(t *testing.T) {
|
|
require.False(t, r.testAndSet("example.com"))
|
|
})
|
|
|
|
t.Run("different key returns true", func(t *testing.T) {
|
|
require.True(t, r.testAndSet("other.com"))
|
|
})
|
|
|
|
t.Run("empty string returns false", func(t *testing.T) {
|
|
require.False(t, r.testAndSet(""))
|
|
})
|
|
|
|
t.Run("whitespace-only returns false", func(t *testing.T) {
|
|
require.False(t, r.testAndSet(" "))
|
|
})
|
|
|
|
t.Run("trimmed duplicate returns false", func(t *testing.T) {
|
|
require.False(t, r.testAndSet(" example.com "))
|
|
})
|
|
}
|
|
|
|
func TestRunner_testAndSet_concurrent(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err, "could not create httpx runner")
|
|
|
|
const goroutines = 100
|
|
key := "race-target.com"
|
|
wins := make([]bool, goroutines)
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(goroutines)
|
|
start := make(chan struct{})
|
|
|
|
for i := 0; i < goroutines; i++ {
|
|
go func(idx int) {
|
|
defer wg.Done()
|
|
<-start
|
|
wins[idx] = r.testAndSet(key)
|
|
}(i)
|
|
}
|
|
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
winCount := 0
|
|
for _, w := range wins {
|
|
if w {
|
|
winCount++
|
|
}
|
|
}
|
|
require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key")
|
|
}
|
|
|
|
func TestOptions_hasMatcherOrFilter(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
options Options
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "no matchers or filters",
|
|
options: Options{},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "match status code",
|
|
options: Options{OutputMatchStatusCode: "200"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter status code",
|
|
options: Options{OutputFilterStatusCode: "403,401"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match string",
|
|
options: Options{OutputMatchString: []string{"admin"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter string",
|
|
options: Options{OutputFilterString: []string{"error"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match content length",
|
|
options: Options{OutputMatchContentLength: "100"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter content length",
|
|
options: Options{OutputFilterContentLength: "0"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match regex",
|
|
options: Options{OutputMatchRegex: []string{"admin.*panel"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter regex",
|
|
options: Options{OutputFilterRegex: []string{"error"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match lines count",
|
|
options: Options{OutputMatchLinesCount: "50"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter lines count",
|
|
options: Options{OutputFilterLinesCount: "0"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match words count",
|
|
options: Options{OutputMatchWordsCount: "100"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter words count",
|
|
options: Options{OutputFilterWordsCount: "0"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match favicon",
|
|
options: Options{OutputMatchFavicon: []string{"1494302000"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter favicon",
|
|
options: Options{OutputFilterFavicon: []string{"1494302000"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match cdn",
|
|
options: Options{OutputMatchCdn: []string{"cloudflare"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter cdn",
|
|
options: Options{OutputFilterCdn: []string{"cloudflare"}},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match condition",
|
|
options: Options{OutputMatchCondition: "status_code == 200"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter condition",
|
|
options: Options{OutputFilterCondition: "status_code == 403"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "match response time",
|
|
options: Options{OutputMatchResponseTime: "< 1"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter response time",
|
|
options: Options{OutputFilterResponseTime: "> 5"},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "filter page type",
|
|
options: Options{OutputFilterPageType: []string{"error"}},
|
|
expected: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
opts := tc.options
|
|
err := opts.ValidateOptions()
|
|
require.Nil(t, err)
|
|
require.Equal(t, tc.expected, opts.HasMatcherOrFilter(),
|
|
"HasMatcherOrFilter() should be %v for %s", tc.expected, tc.name)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
|
|
dir := t.TempDir()
|
|
opts := &Options{
|
|
StoreResponse: true,
|
|
StoreResponseDir: dir,
|
|
}
|
|
err := opts.ValidateOptions()
|
|
require.Nil(t, err)
|
|
require.False(t, opts.HasMatcherOrFilter())
|
|
}
|
|
|
|
func TestStoreResponse_withMatcherSetsFlag(t *testing.T) {
|
|
dir := t.TempDir()
|
|
opts := &Options{
|
|
StoreResponse: true,
|
|
StoreResponseDir: dir,
|
|
OutputMatchStatusCode: "200",
|
|
}
|
|
err := opts.ValidateOptions()
|
|
require.Nil(t, err)
|
|
require.True(t, opts.HasMatcherOrFilter())
|
|
}
|
|
|
|
func TestRunner_duplicate(t *testing.T) {
|
|
const (
|
|
pageA = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here</body></html>"
|
|
pageB = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Dashboard</title></head><body>Completely different application running on this server</body></html>"
|
|
)
|
|
|
|
t.Run("same content same IP is duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://b.example.com"}
|
|
|
|
require.False(t, r.duplicate(first), "first result should not be duplicate")
|
|
require.True(t, r.duplicate(second), "same content + same IP should be duplicate")
|
|
})
|
|
|
|
t.Run("same content different IP is NOT duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.False(t, r.duplicate(second), "same content but different IP should NOT be duplicate")
|
|
})
|
|
|
|
t.Run("different content same IP is NOT duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageB, HostIP: "1.1.1.1", URL: "https://b.example.com"}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.False(t, r.duplicate(second), "different content on same IP should NOT be duplicate")
|
|
})
|
|
|
|
t.Run("different content different IP is NOT duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageB, HostIP: "2.2.2.2", URL: "https://b.example.com"}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.False(t, r.duplicate(second), "different content + different IP should NOT be duplicate")
|
|
})
|
|
|
|
t.Run("third subdomain same content same IP is duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageA, HostIP: "2.2.2.2", URL: "https://b.example.com"}
|
|
third := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://c.example.com"}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.False(t, r.duplicate(second), "different IP should be kept")
|
|
require.True(t, r.duplicate(third), "same content + same IP as first should be duplicate")
|
|
})
|
|
|
|
t.Run("near-duplicate content same IP is duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
nearDup := &Result{
|
|
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
|
|
HostIP: "1.1.1.1",
|
|
URL: "https://b.example.com",
|
|
}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.True(t, r.duplicate(nearDup), "near-duplicate content from same IP should be duplicate")
|
|
})
|
|
|
|
t.Run("near-duplicate content different IP is NOT duplicate", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "1.1.1.1", URL: "https://a.example.com"}
|
|
nearDup := &Result{
|
|
Raw: "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here!</body></html>",
|
|
HostIP: "3.3.3.3",
|
|
URL: "https://b.example.com",
|
|
}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.False(t, r.duplicate(nearDup), "near-duplicate content from different IP should NOT be duplicate")
|
|
})
|
|
|
|
t.Run("empty IP falls back to content-only dedup", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
first := &Result{Raw: pageA, HostIP: "", URL: "https://a.example.com"}
|
|
second := &Result{Raw: pageA, HostIP: "", URL: "https://b.example.com"}
|
|
|
|
require.False(t, r.duplicate(first))
|
|
require.True(t, r.duplicate(second), "empty IP should fall back to content-only dedup")
|
|
})
|
|
|
|
t.Run("many subdomains same default page same IP", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
kept := 0
|
|
for i := 0; i < 50; i++ {
|
|
res := &Result{
|
|
Raw: pageA,
|
|
HostIP: "10.0.0.1",
|
|
URL: fmt.Sprintf("https://sub%d.example.com", i),
|
|
}
|
|
if !r.duplicate(res) {
|
|
kept++
|
|
}
|
|
}
|
|
require.Equal(t, 1, kept, "50 subdomains with identical content on same IP should keep exactly 1")
|
|
})
|
|
|
|
t.Run("many subdomains same default page different IPs", func(t *testing.T) {
|
|
r, err := New(&Options{})
|
|
require.Nil(t, err)
|
|
|
|
kept := 0
|
|
for i := 0; i < 50; i++ {
|
|
res := &Result{
|
|
Raw: pageA,
|
|
HostIP: fmt.Sprintf("10.0.0.%d", i+1),
|
|
URL: fmt.Sprintf("https://sub%d.example.com", i),
|
|
}
|
|
if !r.duplicate(res) {
|
|
kept++
|
|
}
|
|
}
|
|
require.Equal(t, 50, kept, "50 subdomains with identical content but different IPs should keep all 50")
|
|
})
|
|
}
|
|
|
|
func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) {
|
|
runner := &Runner{}
|
|
|
|
tests := []struct {
|
|
name string
|
|
allow []string
|
|
deny []string
|
|
testCases []struct {
|
|
ip string
|
|
expected bool
|
|
reason string
|
|
}
|
|
}{
|
|
{
|
|
name: "Allow flag blocks IPs outside allowed range",
|
|
allow: []string{"192.168.1.0/24"},
|
|
deny: nil,
|
|
testCases: []struct {
|
|
ip string
|
|
expected bool
|
|
reason string
|
|
}{
|
|
{"8.8.8.8", false, "IP outside allowed range should be blocked"},
|
|
{"192.168.1.10", true, "IP inside allowed range should be allowed"},
|
|
},
|
|
},
|
|
{
|
|
name: "Deny flag blocks IPs in denied range",
|
|
allow: nil,
|
|
deny: []string{"127.0.0.0/8"},
|
|
testCases: []struct {
|
|
ip string
|
|
expected bool
|
|
reason string
|
|
}{
|
|
{"127.0.0.1", false, "IP in denied range should be blocked"},
|
|
{"8.8.8.8", true, "IP outside denied range should be allowed"},
|
|
},
|
|
},
|
|
{
|
|
name: "Combined Allow and Deny flags",
|
|
allow: []string{"192.168.0.0/16"},
|
|
deny: []string{"192.168.1.0/24"},
|
|
testCases: []struct {
|
|
ip string
|
|
expected bool
|
|
reason string
|
|
}{
|
|
{"10.0.0.1", false, "IP outside allowed range should be blocked"},
|
|
{"192.168.1.100", false, "IP in denied range should be blocked even if in allowed range"},
|
|
{"192.168.2.50", true, "IP in allowed range but not in denied range should be allowed"},
|
|
},
|
|
},
|
|
{
|
|
name: "Multiple Allow and Deny ranges",
|
|
allow: []string{"10.0.0.0/8", "172.16.0.0/12"},
|
|
deny: []string{"10.1.0.0/16", "172.20.0.0/16"},
|
|
testCases: []struct {
|
|
ip string
|
|
expected bool
|
|
reason string
|
|
}{
|
|
{"10.0.1.1", true, "10.0.1.1 should be allowed (in allow range, not in deny)"},
|
|
{"10.1.1.1", false, "10.1.1.1 should be blocked (in deny range)"},
|
|
{"172.16.1.1", true, "172.16.1.1 should be allowed (in allow range, not in deny)"},
|
|
{"172.20.1.1", false, "172.20.1.1 should be blocked (in deny range)"},
|
|
{"192.168.1.1", false, "192.168.1.1 should be blocked (not in any allow range)"},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
options := &Options{
|
|
Allow: tc.allow,
|
|
Deny: tc.deny,
|
|
}
|
|
|
|
np, err := runner.createNetworkpolicyInstance(options)
|
|
require.Nil(t, err, "could not create networkpolicy instance")
|
|
require.NotNil(t, np, "networkpolicy instance should not be nil")
|
|
|
|
for _, testCase := range tc.testCases {
|
|
allowed := np.Validate(testCase.ip)
|
|
require.Equal(t, testCase.expected, allowed, testCase.reason)
|
|
}
|
|
})
|
|
}
|
|
}
|