Merge branch 'dev' into pr/2219

.github/workflows/stale.yaml

@@ -1,21 +1,41 @@
-name: 'Close stale issues and PR'
+name: 💤 Stale
+
 on:
   schedule:
-    - cron: '30 1 * * *'
+    - cron: "0 0 * * 0" # Weekly
 
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: write # only for delete-branch option
+      issues: write
+      pull-requests: write
     steps:
       - uses: actions/stale@v9
         with:
-          only-labels: "Status: Abandoned, Type: Question"
-          stale-issue-label: stale
-          stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Remove stale label or comment or this will be closed in 90 days.'
-          close-issue-message: 'This issue was closed because it has been stalled for 90 days with no activity.'
-          days-before-stale: 14
-          days-before-close: 90
-          days-before-pr-stale: -1
-          days-before-pr-close: -1
-          stale-pr-message: ''
-          close-pr-message: ''
+          days-before-stale: 90
+          days-before-close: 7
+          stale-issue-label: "Status: Stale"
+          stale-pr-label: "Status: Stale"
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not
+            had recent activity. It will be closed in 7 days if no further
+            activity occurs. Thank you for your contributions!
+          stale-pr-message: >
+            This pull request has been automatically marked as stale due to
+            inactivity. It will be closed in 7 days if no further activity
+            occurs. Please update if you wish to keep it open.
+          close-issue-message: >
+            This issue has been automatically closed due to inactivity. If you
+            think this is a mistake or would like to continue the discussion,
+            please comment or feel free to reopen it.
+          close-pr-message: >
+            This pull request has been automatically closed due to inactivity.
+            If you think this is a mistake or would like to continue working on
+            it, please comment or feel free to reopen it.
+          close-issue-label: "Status: Abandoned"
+          close-pr-label: "Status: Abandoned"
+          exempt-issue-labels: "Status: Abandoned"
+          exempt-pr-labels: "Status: Abandoned"

README.md

@@ -85,10 +85,6 @@ This will display help for the tool. Here are all the switches it supports.
 
 
 ```console
-Usage:
-  ./httpx [flags]
-
-Flags:
 httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.
 
 Usage:
@@ -115,6 +111,7 @@ PROBES:
    -bp, -body-preview     display first N characters of response body (default 100)
    -server, -web-server   display server name
    -td, -tech-detect      display technology in use based on wappalyzer dataset
+   -cff, -custom-fingerprint-file string  path to a custom fingerprint file for technology detection
    -method                display http request method
    -websocket             display server using websocket
    -ip                    display host ip
@@ -203,6 +200,8 @@ OUTPUT:
    -svrc, -store-vision-recon-cluster     include visual recon clusters (-ss and -sr only)
    -pr, -protocol string                  protocol to use (unknown, http11)
    -fepp, -filter-error-page-path string  path to store filtered error pages (default "filtered_error_page.json")
+   -lof, -list-output-fields              list available output field names for filtering
+   -eof, -exclude-output-fields string[]  exclude specified output fields from results
 
 CONFIGURATIONS:
    -config string                   path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
@@ -211,6 +210,7 @@ CONFIGURATIONS:
    -deny string[]                   denied list of IP/CIDR's to process (file or comma separated)
    -sni, -sni-name string           custom TLS SNI name
    -random-agent                    enable Random User-Agent to use (default true)
+   -auto-referer                    set the Referer header to the current URL (default false)
    -H, -header string[]             custom http headers to send with request
    -http-proxy, -proxy string       http proxy to use (eg http://127.0.0.1:8080)
    -unsafe                          send raw requests skipping golang normalization

cmd/functional-test/main.go

@@ -41,7 +41,9 @@ func runFunctionalTests() error {
 	if err != nil {
 		return errors.Wrap(err, "could not open test cases")
 	}
-	defer file.Close()
+	defer func() {
+		_ = file.Close()
+	}()
 
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {

cmd/integration-test/http.go

@@ -49,7 +49,7 @@ type standardHttpGet struct {
 func (h *standardHttpGet) Execute() error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		fmt.Fprintf(w, "This is a test")
+		_, _ = fmt.Fprintf(w, "This is a test")
 		r.Close = true
 	}))
 	var ts *httptest.Server
@@ -100,7 +100,7 @@ func (h *issue276) Execute() error {
 	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		w.Header().Add("Location", ts.URL+"/redirect")
 		w.WriteHeader(302)
-		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+		_, _ = fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -163,7 +163,7 @@ func (h *issue303) Execute() error {
 		// mimic a misconfigured web server behavior declaring gzip body
 		w.Header().Add("Content-Encoding", "gzip")
 		// but sending it uncompressed
-		fmt.Fprint(w, "<html><body>This is a test</body></html>")
+		_, _ = fmt.Fprint(w, "<html><body>This is a test</body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -191,7 +191,7 @@ func (h *issue363) Execute() error {
 	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		w.Header().Add("Location", ts.URL+"/redirect")
 		w.WriteHeader(302)
-		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+		_, _ = fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -214,7 +214,7 @@ func (h *issue400) Execute() error {
 	router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		data, _ := io.ReadAll(r.Body)
-		fmt.Fprintf(w, "data received %s", data)
+		_, _ = fmt.Fprintf(w, "data received %s", data)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -238,7 +238,7 @@ func (h *issue414) Execute() error {
 	router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		data, _ := io.ReadAll(r.Body)
-		fmt.Fprintf(w, "data received %s", data)
+		_, _ = fmt.Fprintf(w, "data received %s", data)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -265,7 +265,7 @@ func (h *titleUnwantedChars) Execute() error {
 	uriPath := "/index"
 	router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		htmlResponse := "<html><head><title>\v\fProject\n\r Discovery\n - Httpx\t></title></head><body>test data</body></html>"
-		fmt.Fprint(w, htmlResponse)
+		_, _ = fmt.Fprint(w, htmlResponse)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -290,7 +290,7 @@ func (h *issue480) Execute() error {
 	uriPath := "////////////////../../../../../../../../etc/passwd"
 	router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		htmlResponse := "<html><body>ok from uri</body></html>"
-		fmt.Fprint(w, htmlResponse)
+		_, _ = fmt.Fprint(w, htmlResponse)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -314,7 +314,7 @@ func (h *customHeader) Execute() error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -341,7 +341,7 @@ func (h *outputMatchCondition) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -365,7 +365,7 @@ func (h *outputFilterCondition) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()
@@ -390,7 +390,7 @@ func (h *outputAll) Execute() error {
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
 		w.Header().Add("Content-Type", "application/json")
 		w.WriteHeader(200)
-		fmt.Fprint(w, `{"status": "ok"}`)
+		_, _ = fmt.Fprint(w, `{"status": "ok"}`)
 	}))
 	ts = httptest.NewServer(router)
 	defer ts.Close()

cmd/integration-test/library.go

@@ -22,7 +22,9 @@ func (h *httpxLibrary) Execute() error {
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(testFile)
+	defer func() {
+		_ = os.RemoveAll(testFile)
+	}()
 
 	var got string
 
@@ -64,7 +66,9 @@ func (h *httpxLibraryWithStream) Execute() error {
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(testFile)
+	defer func() {
+		_ = os.RemoveAll(testFile)
+	}()
 
 	var got string
 

common/customports/customport.go

@@ -77,14 +77,14 @@ func (c *CustomPorts) Set(value string) error {
 			}
 			highP, err := strconv.Atoi(potentialRange[1])
 			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("Could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1]))
+				return errors.Wrap(err, fmt.Sprintf("could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1]))
 			}
 			if err := checkPortValue(highP); err != nil {
 				return errors.Wrap(err, fmt.Sprintf("last port of your range(%d)", lowP))
 			}
 
 			if lowP > highP {
-				return fmt.Errorf("First value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP)
+				return fmt.Errorf("first value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP)
 			}
 
 			for i := lowP; i <= highP; i++ {

common/httpx/httpx.go

@@ -155,7 +155,7 @@ func New(options *Options) (*HTTPX, error) {
 
 	if httpx.Options.Protocol == "http11" {
 		// disable http2
-		os.Setenv("GODEBUG", "http2client=0")
+		_ = os.Setenv("GODEBUG", "http2client=0")
 		transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
 	}
 
@@ -359,7 +359,7 @@ func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOpti
 func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) {
 	method := req.Method
 	headers := req.Header
-	targetURL := req.URL.String()
+	targetURL := req.String()
 	body := req.Body
 	options := rawhttp.DefaultOptions
 	options.Timeout = h.Options.Timeout
@@ -435,6 +435,9 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]st
 		userAgent := useragent.PickRandom()
 		r.Header.Set("User-Agent", userAgent.Raw) //nolint
 	}
+	if h.Options.AutoReferer && r.Header.Get("Referer") == "" {
+		r.Header.Set("Referer", r.String())
+	}
 }
 
 func (httpx *HTTPX) setCustomCookies(req *http.Request) {
@@ -448,7 +451,7 @@ func (httpx *HTTPX) setCustomCookies(req *http.Request) {
 func (httpx *HTTPX) Sanitize(respStr string, trimLine, normalizeSpaces bool) string {
 	respStr = httpx.htmlPolicy.Sanitize(respStr)
 	if trimLine {
-		respStr = strings.Replace(respStr, "\n", "", -1)
+		respStr = strings.ReplaceAll(respStr, "\n", "")
 	}
 	if normalizeSpaces {
 		respStr = httputilz.NormalizeSpaces(respStr)

common/httpx/option.go

@@ -12,6 +12,7 @@ import (
 // Options contains configuration options for the client
 type Options struct {
 	RandomAgent      bool
+	AutoReferer      bool
 	DefaultUserAgent string
 	Proxy            string
 	// Deprecated: use Proxy

common/pagetypeclassifier/pagetypeclassifier.go

@@ -3,7 +3,7 @@ package pagetypeclassifier
 import (
 	_ "embed"
 
-	"github.com/jaytaylor/html2text"
+	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
 	"github.com/projectdiscovery/utils/ml/naive_bayes"
 )
 
@@ -31,7 +31,7 @@ func (n *PageTypeClassifier) Classify(html string) string {
 }
 
 func htmlToText(html string) string {
-	text, err := html2text.FromString(html, html2text.Options{TextOnly: true})
+	text, err := htmltomarkdown.ConvertString(html)
 	if err != nil {
 		panic(err)
 	}

examples/speed_control/main.go

@@ -60,7 +60,9 @@ func main() {
 		if err != nil {
 			log.Fatalf("Error sending PUT request: %v", err)
 		}
-		defer resp.Body.Close()
+		defer func() {
+			_ = resp.Body.Close()
+		}()
 
 		if resp.StatusCode != http.StatusOK {
 			log.Printf("Failed to update threads, status code: %d", resp.StatusCode)

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/hdm/jarm-go v0.0.7
-	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
@@ -22,44 +21,46 @@ require (
 	github.com/miekg/dns v1.1.62 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/asnmap v1.1.1
-	github.com/projectdiscovery/cdncheck v1.1.28
+	github.com/projectdiscovery/cdncheck v1.1.35
 	github.com/projectdiscovery/clistats v0.1.1
-	github.com/projectdiscovery/dsl v0.5.0
-	github.com/projectdiscovery/fastdialer v0.4.2
+	github.com/projectdiscovery/dsl v0.6.0
+	github.com/projectdiscovery/fastdialer v0.4.9
 	github.com/projectdiscovery/fdmax v0.0.4
 	github.com/projectdiscovery/goconfig v0.0.1
 	github.com/projectdiscovery/goflags v0.1.74
 	github.com/projectdiscovery/gologger v1.1.54
-	github.com/projectdiscovery/hmap v0.0.91
+	github.com/projectdiscovery/hmap v0.0.93
 	github.com/projectdiscovery/mapcidr v1.1.34
-	github.com/projectdiscovery/networkpolicy v0.1.18
-	github.com/projectdiscovery/ratelimit v0.0.81
+	github.com/projectdiscovery/networkpolicy v0.1.23
+	github.com/projectdiscovery/ratelimit v0.0.82
 	github.com/projectdiscovery/rawhttp v0.1.90
-	github.com/projectdiscovery/retryablehttp-go v1.0.118
-	github.com/projectdiscovery/tlsx v1.2.0
+	github.com/projectdiscovery/retryablehttp-go v1.0.123
+	github.com/projectdiscovery/tlsx v1.2.1
 	github.com/projectdiscovery/useragent v0.0.101
-	github.com/projectdiscovery/utils v0.4.22
-	github.com/projectdiscovery/wappalyzergo v0.2.38
+	github.com/projectdiscovery/utils v0.5.0
+	github.com/projectdiscovery/wappalyzergo v0.2.45
 	github.com/rs/xid v1.6.0
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 	github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db
 	go.etcd.io/bbolt v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
-	golang.org/x/net v0.42.0
-	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/text v0.27.0
+	golang.org/x/net v0.43.0
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.29.0
 )
 
 require (
+	github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0
 	github.com/go-viper/mapstructure/v2 v2.4.0
 	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
-	github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39
+	github.com/weppos/publicsuffix-go v0.50.0
 )
 
 require (
 	aead.dev/minisign v0.2.0 // indirect
+	github.com/JohannesKaufmann/dom v0.2.0 // indirect
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
@@ -88,7 +89,7 @@ require (
 	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/fgprof v0.9.5 // indirect
-	github.com/gaissmai/bart v0.20.5 // indirect
+	github.com/gaissmai/bart v0.24.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/google/certificate-transparency-go v1.3.2 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
@@ -116,10 +117,9 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
-	github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -127,7 +127,7 @@ require (
 	github.com/projectdiscovery/freeport v0.0.7 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.103 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.106 // indirect
 	github.com/refraction-networking/utls v1.7.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
@@ -136,7 +136,6 @@ require (
 	github.com/shirou/gopsutil/v3 v3.24.2 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sorairolake/lzip-go v0.3.5 // indirect
-	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
@@ -149,25 +148,25 @@ require (
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	github.com/ulikunitz/xz v0.5.12 // indirect
+	github.com/ulikunitz/xz v0.5.15 // indirect
 	github.com/ysmood/fetchup v0.2.3 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
 	github.com/ysmood/got v0.40.0 // indirect
 	github.com/ysmood/gson v0.7.3 // indirect
 	github.com/ysmood/leakless v0.9.0 // indirect
-	github.com/yuin/goldmark v1.7.4 // indirect
+	github.com/yuin/goldmark v1.7.13 // indirect
 	github.com/yuin/goldmark-emoji v1.0.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
-	golang.org/x/crypto v0.40.0 // indirect
-	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/mod v0.27.0 // indirect
 	golang.org/x/oauth2 v0.28.0 // indirect
-	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/term v0.33.0 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/term v0.34.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
-	golang.org/x/tools v0.34.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -20,6 +20,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
+github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
@@ -121,8 +125,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/gaissmai/bart v0.20.5 h1:ehoWZWQ7j//qt0K0Zs4i9hpoPpbgqsMQiR8W2QPJh+c=
-github.com/gaissmai/bart v0.20.5/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
+github.com/gaissmai/bart v0.24.0 h1:HOq5aXDBa4d376KkuxD+xnS9DQWWJtD4zgDNoGV0KrQ=
+github.com/gaissmai/bart v0.24.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68=
 github.com/go-faker/faker/v4 v4.6.1 h1:xUyVpAjEtB04l6XFY0V/29oR332rOSPWV4lU8RwDt4k=
 github.com/go-faker/faker/v4 v4.6.1/go.mod h1:arSdxNCSt7mOhdk8tEolvHeIJ7eX4OX80wXjKKvkKBY=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -214,8 +218,6 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
-github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
-github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -254,7 +256,6 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -278,16 +279,14 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1
 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
-github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg=
-github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ=
+github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
+github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY=
 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
 github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
-github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
-github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
@@ -310,14 +309,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
 github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
-github.com/projectdiscovery/cdncheck v1.1.28 h1:Nn+9SuB4Xv56lFn5zN5hWB0Kq/5veAUX0xnRb+K+IeM=
-github.com/projectdiscovery/cdncheck v1.1.28/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
+github.com/projectdiscovery/cdncheck v1.1.35 h1:xyMnIWf2wzKH4Ii3lBNb73b/n9ee/baEiS3Ao23pyco=
+github.com/projectdiscovery/cdncheck v1.1.35/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
 github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE=
 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
-github.com/projectdiscovery/dsl v0.5.0 h1:3HHY14FNmdwWXq3pi9dd8JjUHQzskZjLD/pZKVx5Vi4=
-github.com/projectdiscovery/dsl v0.5.0/go.mod h1:Fr+zIQJfMNy+RTj5KFgozfvDaiQQEKMyrKXl75aGgxY=
-github.com/projectdiscovery/fastdialer v0.4.2 h1:uB+p8CinbrzlAwfGeTyHKShjnYBj60rcZ8FoBtGlg7I=
-github.com/projectdiscovery/fastdialer v0.4.2/go.mod h1:g3ikfujsn3g+ZdyypNpMzMTj2qHHXXqjZL9YioX53Ok=
+github.com/projectdiscovery/dsl v0.6.0 h1:j4T6WKDx7uuhCSYjvEAySn86G7tTeEEsMeqjUauuqss=
+github.com/projectdiscovery/dsl v0.6.0/go.mod h1:m+tLImvmlEnFTuwb0Bm3DqCL6e7sWOunJyxFi27AduI=
+github.com/projectdiscovery/fastdialer v0.4.9 h1:/CTzoEsayQOrYJjB0CCa/0iNX7LSMeml6XmeyxX8ZVA=
+github.com/projectdiscovery/fastdialer v0.4.9/go.mod h1:/4f7ELi6M5sr5XOpt+q0OpP4sQV36f3bOG23n7GOQUw=
 github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
 github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
 github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -330,32 +329,32 @@ github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJ
 github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
-github.com/projectdiscovery/hmap v0.0.91 h1:8vSTU+3hmMfA5Qd14ceq4j7wnUVUJcXdqQgqbsFBea0=
-github.com/projectdiscovery/hmap v0.0.91/go.mod h1:BxEg8WXqxqaOADPZ+xp2X6BY+znfV8dusCSGSFnsU3c=
+github.com/projectdiscovery/hmap v0.0.93 h1:iIRdioT4byGJ4Hz5cOjo1fd3HFFi1MUFPv+EGYc1yng=
+github.com/projectdiscovery/hmap v0.0.93/go.mod h1:oKgtWo2QMD7BkW25ezYbFCqKO3IctZ2ByEaG2XWW0t0=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
 github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
 github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
-github.com/projectdiscovery/networkpolicy v0.1.18 h1:DAeP73SvcuT4evaohNS7BPELw+VtvcVt4PaTK3fC1qA=
-github.com/projectdiscovery/networkpolicy v0.1.18/go.mod h1:2yWanKsU2oBZ75ch94IsEQy6hByFp+3oTiSyC6ew3TE=
-github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI=
-github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA=
+github.com/projectdiscovery/networkpolicy v0.1.23 h1:+MVm9xHCfzmZG5WhUtjAGFvNiQNKOgxZDDed1QfpLXI=
+github.com/projectdiscovery/networkpolicy v0.1.23/go.mod h1:ILun9d4jgAfLOYf/NYjV+sKyW3tZTLMl+HyvaZuXZo0=
+github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak=
+github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM=
 github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw=
 github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y=
-github.com/projectdiscovery/retryabledns v1.0.103 h1:rPnoMTK+CXLbO8kT7ODtwbhyQGAUpJsqhVq8AAvu1bs=
-github.com/projectdiscovery/retryabledns v1.0.103/go.mod h1:sfu91YrZkb8Ccvij8YDTV96cQt69IPqnfa+OEFUke1o=
-github.com/projectdiscovery/retryablehttp-go v1.0.118 h1:ylbb2mKOfFQOwbtGyI1C0yzudl/UvJvC9YbWIPMyXQY=
-github.com/projectdiscovery/retryablehttp-go v1.0.118/go.mod h1:pAQWFh6lg9Gmno5zrQxbfuAbc9OvIugl5P9kaoXztgM=
+github.com/projectdiscovery/retryabledns v1.0.106 h1:repm5aGq5ge6fNJQbPNux7vyUhtUju7aIMEXZD9FX0A=
+github.com/projectdiscovery/retryabledns v1.0.106/go.mod h1:1W3ogENvt3Pb+Ju4AN0aE2lZy+D76Yzxsz51DqFCIk8=
+github.com/projectdiscovery/retryablehttp-go v1.0.123 h1:jnbNlYJ64ScyMIiPNpyXXX7fVpOfoII20knO2xHH+4s=
+github.com/projectdiscovery/retryablehttp-go v1.0.123/go.mod h1:lv7L70Q/Lc+LARjk6ji+Ihd4Bjb9c/qmqLcdCH8Gzbo=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
-github.com/projectdiscovery/tlsx v1.2.0 h1:n92B+ZAIzWW3+UFOJpI4ILWZuDcQHHLwVqbWXnZPJdc=
-github.com/projectdiscovery/tlsx v1.2.0/go.mod h1:ddPZIluCZBN4N0nn+H0f/QjimF7MOhpQ5RXkJ7zZnwA=
+github.com/projectdiscovery/tlsx v1.2.1 h1:R8QgKb/vxd6Y0cfGFBYs4nn0zodHABeeLPqJjs2mNrA=
+github.com/projectdiscovery/tlsx v1.2.1/go.mod h1:p19UHGQ6bvcbvhO4NvYBKOxlE4QvrUaectx9g/Mm3JA=
 github.com/projectdiscovery/useragent v0.0.101 h1:8A+XOJ/nIH+WqW8ogLxJ/psemGp8ATQ2/GuKroJ/81E=
 github.com/projectdiscovery/useragent v0.0.101/go.mod h1:RGoRw1BQ/lJnhYMbMpEKjyAAgCaDCr/+GsULo5yEJ2I=
-github.com/projectdiscovery/utils v0.4.22 h1:OO3FU2uX967sQxu5JtpdBZNzOevvKHAhWqkoTGl+C0A=
-github.com/projectdiscovery/utils v0.4.22/go.mod h1:3l84gpCwL9KG1/ZmslOBABCrk84CcpGWJZfR8wZysR4=
-github.com/projectdiscovery/wappalyzergo v0.2.38 h1:R7gf5NlndtKd2OFT34uIgFlXDJzE8MXRIwQandAC40M=
-github.com/projectdiscovery/wappalyzergo v0.2.38/go.mod h1:XQSgnMQnxliVw2wjKehxeJ4QTRluOJpmm55gLHdztVQ=
+github.com/projectdiscovery/utils v0.5.0 h1:DN7mg2DpyObLByuObXzAFEkdNRDoPUnqE5N2szd3b3c=
+github.com/projectdiscovery/utils v0.5.0/go.mod h1:eCAWMmyaNxyPWbiKv1oeYJLIKpxceHE2+NWx3Jodhqk=
+github.com/projectdiscovery/wappalyzergo v0.2.45 h1:tx0UuYw9GjDy/FMLsL9mr3HjPXoa3qS/lFnda/zQvf4=
+github.com/projectdiscovery/wappalyzergo v0.2.45/go.mod h1:1dHfTJRhrbbWBdKwl1p4QKpUDNnPYlHBBL7rEwCDdjM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0=
 github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ=
@@ -373,6 +372,10 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY=
 github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E=
+github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
+github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
+github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
 github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@@ -386,8 +389,6 @@ github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu
 github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
-github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -400,8 +401,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
@@ -433,12 +434,12 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
-github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
+github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
-github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM=
-github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA=
+github.com/weppos/publicsuffix-go v0.50.0 h1:M178k6l8cnh9T1c1cStkhytVxdk5zPd6gGZf8ySIuVo=
+github.com/weppos/publicsuffix-go v0.50.0/go.mod h1:VXhClBYMlDrUsome4pOTpe68Ui0p6iQRAbyHQD1yKoU=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
@@ -459,8 +460,8 @@ github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU
 github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
-github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
+github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
 github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4=
 github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
@@ -505,8 +506,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
-golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -539,8 +540,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
-golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -571,8 +572,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
-golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -594,8 +595,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
-golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -636,8 +637,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -650,8 +651,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
-golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
+golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
+golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -667,8 +668,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
-golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
@@ -702,8 +703,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
-golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/pdcp/writer.go

@@ -18,7 +18,7 @@ import (
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/projectdiscovery/utils/conversion"
 	"github.com/projectdiscovery/utils/env"
-	errorutil "github.com/projectdiscovery/utils/errors"
+	"github.com/projectdiscovery/utils/errkit" //
 	unitutils "github.com/projectdiscovery/utils/unit"
 	updateutils "github.com/projectdiscovery/utils/update"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -71,7 +71,7 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential
 	var err error
 	tmp, err := urlutil.Parse(creds.Server)
 	if err != nil {
-		return nil, errorutil.NewWithErr(err).Msgf("could not parse server url")
+		return nil, errkit.Wrap(err, "could not parse server url")
 	}
 	tmp.Path = uploadEndpoint
 	tmp.Update()
@@ -186,7 +186,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context) {
 // uploadChunk uploads a chunk of data to the server
 func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error {
 	if err := u.upload(buff.Bytes()); err != nil {
-		return errorutil.NewWithErr(err).Msgf("could not upload chunk")
+		return errkit.Wrap(err, "could not upload chunk")
 	}
 	// if successful, reset the buffer
 	buff.Reset()
@@ -198,23 +198,25 @@ func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error {
 func (u *UploadWriter) upload(data []byte) error {
 	req, err := u.getRequest(data)
 	if err != nil {
-		return errorutil.NewWithErr(err).Msgf("could not create upload request")
+		return errkit.Wrap(err, "could not create upload request")
 	}
 	resp, err := u.client.Do(req)
 	if err != nil {
-		return errorutil.NewWithErr(err).Msgf("could not upload results")
+		return errkit.Wrap(err, "could not upload results")
 	}
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	bin, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return errorutil.NewWithErr(err).Msgf("could not get id from response")
+		return errkit.Wrap(err, "could not get id from response")
 	}
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("could not upload results got status code %v on %v", resp.StatusCode, resp.Request.URL.String())
 	}
 	var uploadResp uploadResponse
 	if err := json.Unmarshal(bin, &uploadResp); err != nil {
-		return errorutil.NewWithErr(err).Msgf("could not unmarshal response got %v", string(bin))
+		return errkit.Wrapf(err, "could not unmarshal response got %v", string(bin))
 	}
 	if uploadResp.ID != "" && u.assetGroupID == "" {
 		u.assetGroupID = uploadResp.ID
@@ -239,15 +241,15 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
 	}
 	req, err := retryablehttp.NewRequest(method, url, bytes.NewReader(bin))
 	if err != nil {
-		return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request")
+		return nil, errkit.Wrap(err, "could not create cloud upload request")
 	}
 	// add pdtm meta params
-	req.URL.Params.Merge(updateutils.GetpdtmParams(runner.Version))
+	req.Params.Merge(updateutils.GetpdtmParams(runner.Version))
 	// if it is upload endpoint also include name if it exists
-	if u.assetGroupName != "" && req.URL.Path == uploadEndpoint {
-		req.URL.Params.Add("name", u.assetGroupName)
+	if u.assetGroupName != "" && req.Path == uploadEndpoint {
+		req.Params.Add("name", u.assetGroupName)
 	}
-	req.URL.Update()
+	req.Update()
 
 	req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
 	if u.TeamID != "" {

runner/headless.go

@@ -123,7 +123,9 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim
 	}
 
 	page = page.Timeout(timeout)
-	defer page.Close()
+	defer func() {
+		_ = page.Close()
+	}()
 
 	if err := page.Navigate(url); err != nil {
 		return nil, "", err
@@ -150,7 +152,7 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim
 }
 
 func (b *Browser) Close() {
-	b.engine.Close()
-	os.RemoveAll(b.tempDir)
+	_ = b.engine.Close()
+	_ = os.RemoveAll(b.tempDir)
 	// processutil.CloseProcesses(processutil.IsChromeProcess, b.pids)
 }

runner/healthcheck.go

@@ -43,7 +43,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string {
 	test.WriteString(fmt.Sprintf("Config file \"%s\" Write => %s\n", cfgFilePath, testResult))
 	c4, err := net.Dial("tcp4", "scanme.sh:80")
 	if err == nil && c4 != nil {
-		c4.Close()
+		_ = c4.Close()
 	}
 	testResult = "Ok"
 	if err != nil {
@@ -52,7 +52,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string {
 	test.WriteString(fmt.Sprintf("IPv4 connectivity to scanme.sh:80 => %s\n", testResult))
 	c6, err := net.Dial("tcp6", "scanme.sh:80")
 	if err == nil && c6 != nil {
-		c6.Close()
+		_ = c6.Close()
 	}
 	testResult = "Ok"
 	if err != nil {

runner/options.go

@@ -31,6 +31,7 @@ import (
 	fileutil "github.com/projectdiscovery/utils/file"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	"github.com/projectdiscovery/utils/structs"
 	updateutils "github.com/projectdiscovery/utils/update"
 	wappalyzer "github.com/projectdiscovery/wappalyzergo"
 )
@@ -256,11 +257,13 @@ type Options struct {
 	NoFallback                bool
 	NoFallbackScheme          bool
 	TechDetect                bool
+	CustomFingerprintFile     string
 	TLSGrab                   bool
 	protocol                  string
 	ShowStatistics            bool
 	StatsInterval             int
 	RandomAgent               bool
+	AutoReferer               bool
 	StoreChain                bool
 	StoreVisionReconClusters  bool
 	Deny                      customlist.CustomList
@@ -310,6 +313,8 @@ type Options struct {
 	OutputFilterCondition     string
 	OutputMatchCondition      string
 	StripFilter               string
+	ListOutputFields          bool
+	ExcludeOutputFields       goflags.StringSlice
 	//The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err.
 	OnResult             OnResultCallback
 	DisableUpdateCheck   bool
@@ -379,6 +384,7 @@ func ParseOptions() *Options {
 		flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
 		flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
 		flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
+		flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"),
 		flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
 		flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
 		flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
@@ -432,6 +438,8 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
 		flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
 		flagSet.DynamicVar(&options.StripFilter, "strip", "html", "strips all tags in response. supported formats: html,xml"),
+		flagSet.BoolVarP(&options.ListOutputFields, "list-output-fields", "lof", false, "list of fields to output (comma separated)"),
+		flagSet.StringSliceVarP(&options.ExcludeOutputFields, "exclude-output-fields", "eof", nil, "exclude output fields output based on a condition", goflags.NormalizedOriginalStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
@@ -473,7 +481,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.ChainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"),
 		flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"),
 		flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"),
-		flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"),
+		flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11, http2 [experimental], http3 [experimental])"),
 		flagSet.StringVarP(&options.OutputFilterErrorPagePath, "filter-error-page-path", "fepp", "filtered_error_page.json", "path to store filtered error pages"),
 	)
 
@@ -484,6 +492,7 @@ func ParseOptions() *Options {
 		flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"),
 		flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"),
 		flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"),
+		flagSet.BoolVar(&options.AutoReferer, "auto-referer", false, "set the Referer header to the current URL"),
 		flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"),
 		flagSet.StringVarP(&options.Proxy, "proxy", "http-proxy", "", "proxy (http|socks) to use (eg http://127.0.0.1:8080)"),
 		flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"),
@@ -544,6 +553,17 @@ func ParseOptions() *Options {
 
 	_ = flagSet.Parse()
 
+	if options.ListOutputFields {
+		fields, err := structs.GetStructFields(Result{})
+		if err != nil {
+			gologger.Fatal().Msgf("Could not get struct fields: %s\n", err)
+		}
+		for _, field := range fields {
+			fmt.Println(field)
+		}
+		os.Exit(0)
+	}
+
 	if options.OutputAll && options.Output == "" {
 		gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all")
 	}

runner/runner.go

@@ -39,8 +39,9 @@ import (
 	"github.com/projectdiscovery/httpx/static"
 	"github.com/projectdiscovery/mapcidr/asn"
 	"github.com/projectdiscovery/networkpolicy"
-	errorutil "github.com/projectdiscovery/utils/errors"
+	errorutil "github.com/projectdiscovery/utils/errors" //nolint
 	osutil "github.com/projectdiscovery/utils/os"
+	"github.com/projectdiscovery/utils/structs"
 
 	"github.com/Mzack9999/gcache"
 	"github.com/logrusorgru/aurora"
@@ -118,15 +119,20 @@ func New(options *Options) (*Runner, error) {
 	if options.Wappalyzer != nil {
 		runner.wappalyzer = options.Wappalyzer
 	} else if options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload {
-		runner.wappalyzer, err = wappalyzer.New()
+		runner.wappalyzer, err = func() (*wappalyzer.Wappalyze, error) {
+			if options.CustomFingerprintFile != "" {
+				return wappalyzer.NewFromFile(options.CustomFingerprintFile, true, true)
+			}
+			return wappalyzer.New()
+		}()
 	}
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create wappalyzer client")
 	}
 
 	if options.StoreResponseDir != "" {
-		os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
-		os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
+		_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
+		_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
 	}
 
 	httpxOptions := httpx.DefaultOptions
@@ -168,6 +174,11 @@ func New(options *Options) (*Runner, error) {
 	} else {
 		httpxOptions.RandomAgent = options.RandomAgent
 	}
+	if options.CustomHeaders.Has("Referer:") {
+		httpxOptions.AutoReferer = false
+	} else {
+		httpxOptions.AutoReferer = options.AutoReferer
+	}
 	httpxOptions.ZTLS = options.ZTLS
 	httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave)
 	httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead)
@@ -177,6 +188,7 @@ func New(options *Options) (*Runner, error) {
 	}
 	httpxOptions.Resolvers = options.Resolvers
 	httpxOptions.TlsImpersonate = options.TlsImpersonate
+	httpxOptions.Protocol = httpx.Proto(options.Protocol)
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)
@@ -412,10 +424,27 @@ func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpol
 			npOptions.DenyList = append(npOptions.DenyList, exclude)
 		}
 	}
+
+	npOptions.AllowList = appendToList(npOptions.AllowList, options.Allow...)
+	npOptions.DenyList = appendToList(npOptions.DenyList, options.Deny...)
+
 	np, err := networkpolicy.New(npOptions)
 	return np, err
 }
 
+func appendToList(list []string, values ...string) []string {
+	for _, value := range values {
+		switch {
+		case asn.IsASN(value):
+			ips := expandASNInputValue(value)
+			list = append(list, ips...)
+		default:
+			list = append(list, value)
+		}
+	}
+	return list
+}
+
 func expandCIDRInputValue(value string) []string {
 	var ips []string
 	ipsCh, _ := mapcidr.IPAddressesAsStream(value)
@@ -673,7 +702,7 @@ func makePrintCallback() func(stats clistats.StatisticsClient) interface{} {
 		builder.WriteString(clistats.String(uint64(incrementRequests / duration.Seconds())))
 
 		builder.WriteString(" | Requests: ")
-		builder.WriteString(fmt.Sprintf("%.0f", currentRequests))
+		_, _ = fmt.Fprintf(builder, "%.0f", currentRequests)
 
 		hosts, _ := stats.GetCounter("hosts")
 		totalHosts, _ := stats.GetStatic("totalHosts")
@@ -784,18 +813,26 @@ func (r *Runner) RunEnumeration() {
 
 		if r.options.Output != "" && r.options.OutputAll {
 			plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
-			defer plainFile.Close()
+			defer func() {
+				_ = plainFile.Close()
+			}()
 			jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json")
-			defer jsonFile.Close()
+			defer func() {
+				_ = jsonFile.Close()
+			}()
 			csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv")
-			defer csvFile.Close()
+			defer func() {
+				_ = csvFile.Close()
+			}()
 		}
 
 		jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput)
 		jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput)
 		if r.options.Output != "" && plainFile == nil && !jsonOrCsv {
 			plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
-			defer plainFile.Close()
+			defer func() {
+				_ = plainFile.Close()
+			}()
 		}
 
 		if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil {
@@ -804,7 +841,9 @@ func (r *Runner) RunEnumeration() {
 				ext = ".json"
 			}
 			jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
-			defer jsonFile.Close()
+			defer func() {
+				_ = jsonFile.Close()
+			}()
 		}
 
 		if r.options.Output != "" && r.options.CSVOutput && csvFile == nil {
@@ -813,7 +852,9 @@ func (r *Runner) RunEnumeration() {
 				ext = ".csv"
 			}
 			csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
-			defer csvFile.Close()
+			defer func() {
+				_ = csvFile.Close()
+			}()
 		}
 
 		if r.options.CSVOutput {
@@ -1076,7 +1117,7 @@ func (r *Runner) RunEnumeration() {
 				// store response
 				if r.scanopts.StoreResponse || r.scanopts.StoreChain {
 					if r.scanopts.OmitBody {
-						resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1)
+						resp.Raw = strings.ReplaceAll(resp.Raw, resp.ResponseBody, "")
 					}
 
 					responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
@@ -1153,6 +1194,12 @@ func (r *Runner) RunEnumeration() {
 				plainFile.WriteString(resp.str + "\n")
 			}
 
+			if len(r.options.ExcludeOutputFields) > 0 {
+				if filteredData, err := structs.FilterStruct(resp, nil, r.options.ExcludeOutputFields); err == nil {
+					resp = filteredData
+				}
+			}
+
 			// call the callback function if any
 			// be careful and check for result.Err
 			if r.options.OnResult != nil {
@@ -1204,7 +1251,9 @@ func (r *Runner) RunEnumeration() {
 			if err != nil {
 				gologger.Warning().Msgf("Could not create HTML file %s\n", err)
 			}
-			defer screenshotHtml.Close()
+			defer func() {
+				_ = screenshotHtml.Close()
+			}()
 
 			templateMap := template.FuncMap{
 				"safeURL": func(u string) template.URL {
@@ -1296,7 +1345,9 @@ func (r *Runner) RunEnumeration() {
 		if err != nil {
 			gologger.Fatal().Msgf("Failed to create JSON file: %v", err)
 		}
-		defer file.Close()
+		defer func() {
+			_ = file.Close()
+		}()
 
 		_, err = file.Write(clusterReportJSON)
 		if err != nil {
@@ -1320,7 +1371,9 @@ func logFilteredErrorPage(fileName, url string) {
 		gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", fileName, err)
 		return
 	}
-	defer file.Close()
+	defer func() {
+		_ = file.Close()
+	}()
 
 	info := map[string]interface{}{
 		"url":           url,
@@ -1621,7 +1674,7 @@ retry:
 	var requestDump []byte
 	if scanopts.Unsafe {
 		var errDump error
-		requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
+		requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
 		if errDump != nil {
 			return Result{URL: URL.String(), Input: origInput, Err: errDump}
 		}
@@ -1644,7 +1697,7 @@ retry:
 		}
 	}
 	// fix the final output url
-	fullURL := req.URL.String()
+	fullURL := req.String()
 	if parsedURL, errParse := r.parseURL(fullURL); errParse != nil {
 		return Result{URL: URL.String(), Input: origInput, Err: errParse}
 	} else {
@@ -1830,7 +1883,7 @@ retry:
 
 	serverHeader := resp.GetHeader("Server")
 	if scanopts.OutputServerHeader {
-		builder.WriteString(fmt.Sprintf(" [%s]", serverHeader))
+		_, _ = fmt.Fprintf(builder, " [%s]", serverHeader)
 	}
 
 	var (
@@ -1944,7 +1997,7 @@ retry:
 	}
 
 	if scanopts.OutputIP || scanopts.ProbeAllIPS {
-		builder.WriteString(fmt.Sprintf(" [%s]", ip))
+		_, _ = fmt.Fprintf(builder, " [%s]", ip)
 	}
 
 	var onlyHost string
@@ -1969,16 +2022,16 @@ retry:
 
 	if scanopts.OutputCName && len(cnames) > 0 {
 		// Print only the first CNAME (full list in json)
-		builder.WriteString(fmt.Sprintf(" [%s]", cnames[0]))
+		_, _ = fmt.Fprintf(builder, " [%s]", cnames[0])
 	}
 
 	isCDN, cdnName, cdnType, err := hp.CdnCheck(ip)
 	if scanopts.OutputCDN == "true" && isCDN && err == nil {
-		builder.WriteString(fmt.Sprintf(" [%s]", cdnName))
+		_, _ = fmt.Fprintf(builder, " [%s]", cdnName)
 	}
 
 	if scanopts.OutputResponseTime {
-		builder.WriteString(fmt.Sprintf(" [%s]", resp.Duration))
+		_, _ = fmt.Fprintf(builder, " [%s]", resp.Duration)
 	}
 
 	technologyDetails := make(map[string]wappalyzer.AppInfo)
@@ -2042,7 +2095,7 @@ retry:
 	hashesMap := make(map[string]interface{})
 	if scanopts.Hashes != "" {
 		hs := strings.Split(scanopts.Hashes, ",")
-		outputHashes := !(r.options.JSONOutput || r.options.OutputAll)
+		outputHashes := !(r.options.JSONOutput || r.options.OutputAll) //nolint
 		if outputHashes {
 			builder.WriteString(" [")
 		}
@@ -2095,7 +2148,7 @@ retry:
 		if !scanopts.OutputWithNoColor {
 			builder.WriteString(aurora.Magenta(resp.Lines).String())
 		} else {
-			builder.WriteString(fmt.Sprint(resp.Lines))
+			_, _ = fmt.Fprintf(builder, "%d", resp.Lines)
 		}
 		builder.WriteRune(']')
 	}
@@ -2106,7 +2159,7 @@ retry:
 		if !scanopts.OutputWithNoColor {
 			builder.WriteString(aurora.Magenta(jarmhash).String())
 		} else {
-			builder.WriteString(fmt.Sprint(jarmhash))
+			_, _ = fmt.Fprintf(builder, "%s", jarmhash)
 		}
 		builder.WriteRune(']')
 	}
@@ -2115,7 +2168,7 @@ retry:
 		if !scanopts.OutputWithNoColor {
 			builder.WriteString(aurora.Magenta(resp.Words).String())
 		} else {
-			builder.WriteString(fmt.Sprint(resp.Words))
+			_, _ = fmt.Fprintf(builder, "%d", resp.Words)
 		}
 		builder.WriteRune(']')
 	}
@@ -2133,7 +2186,7 @@ retry:
 	// store response
 	if scanopts.StoreResponse || scanopts.StoreChain {
 		if r.options.OmitBody {
-			resp.Raw = strings.Replace(resp.Raw, string(resp.Data), "", -1)
+			resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "")
 		}
 		responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
 		// URL.EscapedString returns that can be used as filename
@@ -2336,9 +2389,9 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) {
 
 func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) {
 	// Check if current URI is ending with .ico => use current body without additional requests
-	if path.Ext(req.URL.Path) == ".ico" {
+	if path.Ext(req.Path) == ".ico" {
 		mmh3, md5h, err := r.calculateFaviconHashWithRaw(currentResp)
-		return mmh3, md5h, req.URL.Path, currentResp, req.URL.String(), err
+		return mmh3, md5h, req.Path, currentResp, req.String(), err
 	}
 
 	// Parse HTML: collect <link rel="...icon..."> hrefs + optional <base href>
@@ -2353,7 +2406,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 	}
 
 	// Determine base URL: prefer finalURL (redirect target) then apply <base href>
-	baseNet, _ := url.Parse(req.URL.String())
+	baseNet, _ := url.Parse(req.String())
 	if finalURL != "" {
 		if u, err := url.Parse(finalURL); err == nil {
 			baseNet = u
@@ -2441,7 +2494,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 		faviconMMH3 = mmh3
 		faviconMD5 = md5h
 		faviconPath = raw
-		faviconURL = clone.URL.String()
+		faviconURL = clone.String()
 		faviconData = respFav.Data
 		gologger.Debug().Msgf("favicon resolved url=%s raw_href=%s size=%d bytes", faviconURL, faviconPath, len(faviconData))
 		break
@@ -2453,7 +2506,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, string, error) {
 	hashNum, md5Hash, err := stringz.FaviconHash(data)
 	if err != nil {
-		return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err)
+		return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) //nolint
 	}
 	return fmt.Sprintf("%d", hashNum), md5Hash, nil
 }

runner/runner_test.go

@@ -222,3 +222,93 @@ func TestRunner_CSVRow(t *testing.T) {
 		t.Error("CSV sanitization incorrectly modified non-vulnerable field")
 	}
 }
+
+func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) {
+	runner := &Runner{}
+
+	tests := []struct {
+		name        string
+		allow       []string
+		deny        []string
+		testCases   []struct {
+			ip       string
+			expected bool
+			reason   string
+		}
+	}{
+		{
+			name:  "Allow flag blocks IPs outside allowed range",
+			allow: []string{"192.168.1.0/24"},
+			deny:  nil,
+			testCases: []struct {
+				ip       string
+				expected bool
+				reason   string
+			}{
+				{"8.8.8.8", false, "IP outside allowed range should be blocked"},
+				{"192.168.1.10", true, "IP inside allowed range should be allowed"},
+			},
+		},
+		{
+			name:  "Deny flag blocks IPs in denied range",
+			allow: nil,
+			deny:  []string{"127.0.0.0/8"},
+			testCases: []struct {
+				ip       string
+				expected bool
+				reason   string
+			}{
+				{"127.0.0.1", false, "IP in denied range should be blocked"},
+				{"8.8.8.8", true, "IP outside denied range should be allowed"},
+			},
+		},
+		{
+			name:  "Combined Allow and Deny flags",
+			allow: []string{"192.168.0.0/16"},
+			deny:  []string{"192.168.1.0/24"},
+			testCases: []struct {
+				ip       string
+				expected bool
+				reason   string
+			}{
+				{"10.0.0.1", false, "IP outside allowed range should be blocked"},
+				{"192.168.1.100", false, "IP in denied range should be blocked even if in allowed range"},
+				{"192.168.2.50", true, "IP in allowed range but not in denied range should be allowed"},
+			},
+		},
+		{
+			name:  "Multiple Allow and Deny ranges",
+			allow: []string{"10.0.0.0/8", "172.16.0.0/12"},
+			deny:  []string{"10.1.0.0/16", "172.20.0.0/16"},
+			testCases: []struct {
+				ip       string
+				expected bool
+				reason   string
+			}{
+				{"10.0.1.1", true, "10.0.1.1 should be allowed (in allow range, not in deny)"},
+				{"10.1.1.1", false, "10.1.1.1 should be blocked (in deny range)"},
+				{"172.16.1.1", true, "172.16.1.1 should be allowed (in allow range, not in deny)"},
+				{"172.20.1.1", false, "172.20.1.1 should be blocked (in deny range)"},
+				{"192.168.1.1", false, "192.168.1.1 should be blocked (not in any allow range)"},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			options := &Options{
+				Allow: tc.allow,
+				Deny:  tc.deny,
+			}
+
+			np, err := runner.createNetworkpolicyInstance(options)
+			require.Nil(t, err, "could not create networkpolicy instance")
+			require.NotNil(t, np, "networkpolicy instance should not be nil")
+
+			for _, testCase := range tc.testCases {
+				allowed := np.Validate(testCase.ip)
+				require.Equal(t, testCase.expected, allowed, testCase.reason)
+			}
+		})
+	}
+}

snapcraft.yaml

@@ -2,7 +2,7 @@ name: httpx
 summary: httpx is a fast and multi-purpose HTTP toolkit
 description: |
   httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads.
-version: 'v1.2.1'
+version: 'v1.7.1'
 icon: static/httpx-logo.png
 license: MIT
 base: core18