diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 636d20d..5254b32 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -1,21 +1,41 @@ -name: 'Close stale issues and PR' +name: 💤 Stale + on: schedule: - - cron: '30 1 * * *' + - cron: "0 0 * * 0" # Weekly jobs: stale: runs-on: ubuntu-latest + permissions: + actions: write + contents: write # only for delete-branch option + issues: write + pull-requests: write steps: - uses: actions/stale@v9 with: - only-labels: "Status: Abandoned, Type: Question" - stale-issue-label: stale - stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Remove stale label or comment or this will be closed in 90 days.' - close-issue-message: 'This issue was closed because it has been stalled for 90 days with no activity.' - days-before-stale: 14 - days-before-close: 90 - days-before-pr-stale: -1 - days-before-pr-close: -1 - stale-pr-message: '' - close-pr-message: '' \ No newline at end of file + days-before-stale: 90 + days-before-close: 7 + stale-issue-label: "Status: Stale" + stale-pr-label: "Status: Stale" + stale-issue-message: > + This issue has been automatically marked as stale because it has not + had recent activity. It will be closed in 7 days if no further + activity occurs. Thank you for your contributions! + stale-pr-message: > + This pull request has been automatically marked as stale due to + inactivity. It will be closed in 7 days if no further activity + occurs. Please update if you wish to keep it open. + close-issue-message: > + This issue has been automatically closed due to inactivity. If you + think this is a mistake or would like to continue the discussion, + please comment or feel free to reopen it. + close-pr-message: > + This pull request has been automatically closed due to inactivity. + If you think this is a mistake or would like to continue working on + it, please comment or feel free to reopen it. + close-issue-label: "Status: Abandoned" + close-pr-label: "Status: Abandoned" + exempt-issue-labels: "Status: Abandoned" + exempt-pr-labels: "Status: Abandoned" diff --git a/README.md b/README.md index efc17d8..5fc010e 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,6 @@ This will display help for the tool. Here are all the switches it supports. ```console -Usage: - ./httpx [flags] - -Flags: httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library. Usage: @@ -115,6 +111,7 @@ PROBES: -bp, -body-preview display first N characters of response body (default 100) -server, -web-server display server name -td, -tech-detect display technology in use based on wappalyzer dataset + -cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection -method display http request method -websocket display server using websocket -ip display host ip @@ -203,6 +200,8 @@ OUTPUT: -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only) -pr, -protocol string protocol to use (unknown, http11) -fepp, -filter-error-page-path string path to store filtered error pages (default "filtered_error_page.json") + -lof, -list-output-fields list available output field names for filtering + -eof, -exclude-output-fields string[] exclude specified output fields from results CONFIGURATIONS: -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) @@ -211,6 +210,7 @@ CONFIGURATIONS: -deny string[] denied list of IP/CIDR's to process (file or comma separated) -sni, -sni-name string custom TLS SNI name -random-agent enable Random User-Agent to use (default true) + -auto-referer set the Referer header to the current URL (default false) -H, -header string[] custom http headers to send with request -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) -unsafe send raw requests skipping golang normalization diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go index 3fd4242..1500ec6 100644 --- a/cmd/functional-test/main.go +++ b/cmd/functional-test/main.go @@ -41,7 +41,9 @@ func runFunctionalTests() error { if err != nil { return errors.Wrap(err, "could not open test cases") } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index caa1448..1017997 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -49,7 +49,7 @@ type standardHttpGet struct { func (h *standardHttpGet) Execute() error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - fmt.Fprintf(w, "This is a test") + _, _ = fmt.Fprintf(w, "This is a test") r.Close = true })) var ts *httptest.Server @@ -100,7 +100,7 @@ func (h *issue276) Execute() error { router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.Header().Add("Location", ts.URL+"/redirect") w.WriteHeader(302) - fmt.Fprintf(w, "Object moved") + _, _ = fmt.Fprintf(w, "Object moved") })) ts = httptest.NewServer(router) defer ts.Close() @@ -163,7 +163,7 @@ func (h *issue303) Execute() error { // mimic a misconfigured web server behavior declaring gzip body w.Header().Add("Content-Encoding", "gzip") // but sending it uncompressed - fmt.Fprint(w, "This is a test") + _, _ = fmt.Fprint(w, "This is a test") })) ts = httptest.NewServer(router) defer ts.Close() @@ -191,7 +191,7 @@ func (h *issue363) Execute() error { router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.Header().Add("Location", ts.URL+"/redirect") w.WriteHeader(302) - fmt.Fprintf(w, "Object moved") + _, _ = fmt.Fprintf(w, "Object moved") })) ts = httptest.NewServer(router) defer ts.Close() @@ -214,7 +214,7 @@ func (h *issue400) Execute() error { router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") data, _ := io.ReadAll(r.Body) - fmt.Fprintf(w, "data received %s", data) + _, _ = fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) defer ts.Close() @@ -238,7 +238,7 @@ func (h *issue414) Execute() error { router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") data, _ := io.ReadAll(r.Body) - fmt.Fprintf(w, "data received %s", data) + _, _ = fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) defer ts.Close() @@ -265,7 +265,7 @@ func (h *titleUnwantedChars) Execute() error { uriPath := "/index" router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { htmlResponse := "\v\fProject\n\r Discovery\n - Httpx\t>test data" - fmt.Fprint(w, htmlResponse) + _, _ = fmt.Fprint(w, htmlResponse) })) ts = httptest.NewServer(router) defer ts.Close() @@ -290,7 +290,7 @@ func (h *issue480) Execute() error { uriPath := "////////////////../../../../../../../../etc/passwd" router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { htmlResponse := "ok from uri" - fmt.Fprint(w, htmlResponse) + _, _ = fmt.Fprint(w, htmlResponse) })) ts = httptest.NewServer(router) defer ts.Close() @@ -314,7 +314,7 @@ func (h *customHeader) Execute() error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") - fmt.Fprint(w, `{"status": "ok"}`) + _, _ = fmt.Fprint(w, `{"status": "ok"}`) })) ts = httptest.NewServer(router) defer ts.Close() @@ -341,7 +341,7 @@ func (h *outputMatchCondition) Execute() error { router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(200) - fmt.Fprint(w, `{"status": "ok"}`) + _, _ = fmt.Fprint(w, `{"status": "ok"}`) })) ts = httptest.NewServer(router) defer ts.Close() @@ -365,7 +365,7 @@ func (h *outputFilterCondition) Execute() error { router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(200) - fmt.Fprint(w, `{"status": "ok"}`) + _, _ = fmt.Fprint(w, `{"status": "ok"}`) })) ts = httptest.NewServer(router) defer ts.Close() @@ -390,7 +390,7 @@ func (h *outputAll) Execute() error { router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(200) - fmt.Fprint(w, `{"status": "ok"}`) + _, _ = fmt.Fprint(w, `{"status": "ok"}`) })) ts = httptest.NewServer(router) defer ts.Close() diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 059fa41..afecc0a 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -22,7 +22,9 @@ func (h *httpxLibrary) Execute() error { if err != nil { return err } - defer os.RemoveAll(testFile) + defer func() { + _ = os.RemoveAll(testFile) + }() var got string @@ -64,7 +66,9 @@ func (h *httpxLibraryWithStream) Execute() error { if err != nil { return err } - defer os.RemoveAll(testFile) + defer func() { + _ = os.RemoveAll(testFile) + }() var got string diff --git a/common/customports/customport.go b/common/customports/customport.go index 9a08def..0e9e5d3 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -77,14 +77,14 @@ func (c *CustomPorts) Set(value string) error { } highP, err := strconv.Atoi(potentialRange[1]) if err != nil { - return errors.Wrap(err, fmt.Sprintf("Could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1])) + return errors.Wrap(err, fmt.Sprintf("could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1])) } if err := checkPortValue(highP); err != nil { return errors.Wrap(err, fmt.Sprintf("last port of your range(%d)", lowP)) } if lowP > highP { - return fmt.Errorf("First value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP) + return fmt.Errorf("first value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP) } for i := lowP; i <= highP; i++ { diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 256066b..d1f0320 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -155,7 +155,7 @@ func New(options *Options) (*HTTPX, error) { if httpx.Options.Protocol == "http11" { // disable http2 - os.Setenv("GODEBUG", "http2client=0") + _ = os.Setenv("GODEBUG", "http2client=0") transport.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} } @@ -359,7 +359,7 @@ func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOpti func (h *HTTPX) doUnsafeWithOptions(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { method := req.Method headers := req.Header - targetURL := req.URL.String() + targetURL := req.String() body := req.Body options := rawhttp.DefaultOptions options.Timeout = h.Options.Timeout @@ -435,6 +435,9 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]st userAgent := useragent.PickRandom() r.Header.Set("User-Agent", userAgent.Raw) //nolint } + if h.Options.AutoReferer && r.Header.Get("Referer") == "" { + r.Header.Set("Referer", r.String()) + } } func (httpx *HTTPX) setCustomCookies(req *http.Request) { @@ -448,7 +451,7 @@ func (httpx *HTTPX) setCustomCookies(req *http.Request) { func (httpx *HTTPX) Sanitize(respStr string, trimLine, normalizeSpaces bool) string { respStr = httpx.htmlPolicy.Sanitize(respStr) if trimLine { - respStr = strings.Replace(respStr, "\n", "", -1) + respStr = strings.ReplaceAll(respStr, "\n", "") } if normalizeSpaces { respStr = httputilz.NormalizeSpaces(respStr) diff --git a/common/httpx/option.go b/common/httpx/option.go index 6386213..b64cfd3 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -12,6 +12,7 @@ import ( // Options contains configuration options for the client type Options struct { RandomAgent bool + AutoReferer bool DefaultUserAgent string Proxy string // Deprecated: use Proxy diff --git a/common/pagetypeclassifier/pagetypeclassifier.go b/common/pagetypeclassifier/pagetypeclassifier.go index 67eb3e0..a1a870b 100644 --- a/common/pagetypeclassifier/pagetypeclassifier.go +++ b/common/pagetypeclassifier/pagetypeclassifier.go @@ -3,7 +3,7 @@ package pagetypeclassifier import ( _ "embed" - "github.com/jaytaylor/html2text" + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/projectdiscovery/utils/ml/naive_bayes" ) @@ -31,7 +31,7 @@ func (n *PageTypeClassifier) Classify(html string) string { } func htmlToText(html string) string { - text, err := html2text.FromString(html, html2text.Options{TextOnly: true}) + text, err := htmltomarkdown.ConvertString(html) if err != nil { panic(err) } diff --git a/examples/speed_control/main.go b/examples/speed_control/main.go index 5e34f6a..ae3d4dd 100644 --- a/examples/speed_control/main.go +++ b/examples/speed_control/main.go @@ -60,7 +60,9 @@ func main() { if err != nil { log.Fatalf("Error sending PUT request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { log.Printf("Failed to update threads, status code: %d", resp.StatusCode) diff --git a/go.mod b/go.mod index c370e6c..e8f608f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/hdm/jarm-go v0.0.7 - github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 @@ -22,44 +21,46 @@ require ( github.com/miekg/dns v1.1.62 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 - github.com/projectdiscovery/cdncheck v1.1.28 + github.com/projectdiscovery/cdncheck v1.1.35 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/dsl v0.5.0 - github.com/projectdiscovery/fastdialer v0.4.2 + github.com/projectdiscovery/dsl v0.6.0 + github.com/projectdiscovery/fastdialer v0.4.9 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.74 github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/hmap v0.0.91 + github.com/projectdiscovery/hmap v0.0.93 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/networkpolicy v0.1.18 - github.com/projectdiscovery/ratelimit v0.0.81 + github.com/projectdiscovery/networkpolicy v0.1.23 + github.com/projectdiscovery/ratelimit v0.0.82 github.com/projectdiscovery/rawhttp v0.1.90 - github.com/projectdiscovery/retryablehttp-go v1.0.118 - github.com/projectdiscovery/tlsx v1.2.0 + github.com/projectdiscovery/retryablehttp-go v1.0.123 + github.com/projectdiscovery/tlsx v1.2.1 github.com/projectdiscovery/useragent v0.0.101 - github.com/projectdiscovery/utils v0.4.22 - github.com/projectdiscovery/wappalyzergo v0.2.38 + github.com/projectdiscovery/utils v0.5.0 + github.com/projectdiscovery/wappalyzergo v0.2.45 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db go.etcd.io/bbolt v1.4.0 // indirect go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 - golang.org/x/net v0.42.0 - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 + golang.org/x/net v0.43.0 + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.29.0 ) require ( + github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 - github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 + github.com/weppos/publicsuffix-go v0.50.0 ) require ( aead.dev/minisign v0.2.0 // indirect + github.com/JohannesKaufmann/dom v0.2.0 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect @@ -88,7 +89,7 @@ require ( github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/fgprof v0.9.5 // indirect - github.com/gaissmai/bart v0.20.5 // indirect + github.com/gaissmai/bart v0.24.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/certificate-transparency-go v1.3.2 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -116,10 +117,9 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -127,7 +127,7 @@ require ( github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.103 // indirect + github.com/projectdiscovery/retryabledns v1.0.106 // indirect github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -136,7 +136,6 @@ require ( github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect - github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.7.0 // indirect @@ -149,25 +148,25 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/got v0.40.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.9.0 // indirect - github.com/yuin/goldmark v1.7.4 // indirect + github.com/yuin/goldmark v1.7.13 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/mod v0.25.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/mod v0.27.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/term v0.33.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/term v0.34.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.34.0 // indirect + golang.org/x/tools v0.36.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index cb4321f..aab78ce 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= @@ -121,8 +125,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.20.5 h1:ehoWZWQ7j//qt0K0Zs4i9hpoPpbgqsMQiR8W2QPJh+c= -github.com/gaissmai/bart v0.20.5/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= +github.com/gaissmai/bart v0.24.0 h1:HOq5aXDBa4d376KkuxD+xnS9DQWWJtD4zgDNoGV0KrQ= +github.com/gaissmai/bart v0.24.0/go.mod h1:RpLtt3lWq1BoRz3AAyDAJ7jhLWBkYhVCfi+ximB2t68= github.com/go-faker/faker/v4 v4.6.1 h1:xUyVpAjEtB04l6XFY0V/29oR332rOSPWV4lU8RwDt4k= github.com/go-faker/faker/v4 v4.6.1/go.mod h1:arSdxNCSt7mOhdk8tEolvHeIJ7eX4OX80wXjKKvkKBY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -214,8 +218,6 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= -github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA= -github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -254,7 +256,6 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -278,16 +279,14 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1 github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= -github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= @@ -310,14 +309,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.28 h1:Nn+9SuB4Xv56lFn5zN5hWB0Kq/5veAUX0xnRb+K+IeM= -github.com/projectdiscovery/cdncheck v1.1.28/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.35 h1:xyMnIWf2wzKH4Ii3lBNb73b/n9ee/baEiS3Ao23pyco= +github.com/projectdiscovery/cdncheck v1.1.35/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.5.0 h1:3HHY14FNmdwWXq3pi9dd8JjUHQzskZjLD/pZKVx5Vi4= -github.com/projectdiscovery/dsl v0.5.0/go.mod h1:Fr+zIQJfMNy+RTj5KFgozfvDaiQQEKMyrKXl75aGgxY= -github.com/projectdiscovery/fastdialer v0.4.2 h1:uB+p8CinbrzlAwfGeTyHKShjnYBj60rcZ8FoBtGlg7I= -github.com/projectdiscovery/fastdialer v0.4.2/go.mod h1:g3ikfujsn3g+ZdyypNpMzMTj2qHHXXqjZL9YioX53Ok= +github.com/projectdiscovery/dsl v0.6.0 h1:j4T6WKDx7uuhCSYjvEAySn86G7tTeEEsMeqjUauuqss= +github.com/projectdiscovery/dsl v0.6.0/go.mod h1:m+tLImvmlEnFTuwb0Bm3DqCL6e7sWOunJyxFi27AduI= +github.com/projectdiscovery/fastdialer v0.4.9 h1:/CTzoEsayQOrYJjB0CCa/0iNX7LSMeml6XmeyxX8ZVA= +github.com/projectdiscovery/fastdialer v0.4.9/go.mod h1:/4f7ELi6M5sr5XOpt+q0OpP4sQV36f3bOG23n7GOQUw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -330,32 +329,32 @@ github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJ github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.91 h1:8vSTU+3hmMfA5Qd14ceq4j7wnUVUJcXdqQgqbsFBea0= -github.com/projectdiscovery/hmap v0.0.91/go.mod h1:BxEg8WXqxqaOADPZ+xp2X6BY+znfV8dusCSGSFnsU3c= +github.com/projectdiscovery/hmap v0.0.93 h1:iIRdioT4byGJ4Hz5cOjo1fd3HFFi1MUFPv+EGYc1yng= +github.com/projectdiscovery/hmap v0.0.93/go.mod h1:oKgtWo2QMD7BkW25ezYbFCqKO3IctZ2ByEaG2XWW0t0= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= -github.com/projectdiscovery/networkpolicy v0.1.18 h1:DAeP73SvcuT4evaohNS7BPELw+VtvcVt4PaTK3fC1qA= -github.com/projectdiscovery/networkpolicy v0.1.18/go.mod h1:2yWanKsU2oBZ75ch94IsEQy6hByFp+3oTiSyC6ew3TE= -github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= -github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= +github.com/projectdiscovery/networkpolicy v0.1.23 h1:+MVm9xHCfzmZG5WhUtjAGFvNiQNKOgxZDDed1QfpLXI= +github.com/projectdiscovery/networkpolicy v0.1.23/go.mod h1:ILun9d4jgAfLOYf/NYjV+sKyW3tZTLMl+HyvaZuXZo0= +github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak= +github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw= github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y= -github.com/projectdiscovery/retryabledns v1.0.103 h1:rPnoMTK+CXLbO8kT7ODtwbhyQGAUpJsqhVq8AAvu1bs= -github.com/projectdiscovery/retryabledns v1.0.103/go.mod h1:sfu91YrZkb8Ccvij8YDTV96cQt69IPqnfa+OEFUke1o= -github.com/projectdiscovery/retryablehttp-go v1.0.118 h1:ylbb2mKOfFQOwbtGyI1C0yzudl/UvJvC9YbWIPMyXQY= -github.com/projectdiscovery/retryablehttp-go v1.0.118/go.mod h1:pAQWFh6lg9Gmno5zrQxbfuAbc9OvIugl5P9kaoXztgM= +github.com/projectdiscovery/retryabledns v1.0.106 h1:repm5aGq5ge6fNJQbPNux7vyUhtUju7aIMEXZD9FX0A= +github.com/projectdiscovery/retryabledns v1.0.106/go.mod h1:1W3ogENvt3Pb+Ju4AN0aE2lZy+D76Yzxsz51DqFCIk8= +github.com/projectdiscovery/retryablehttp-go v1.0.123 h1:jnbNlYJ64ScyMIiPNpyXXX7fVpOfoII20knO2xHH+4s= +github.com/projectdiscovery/retryablehttp-go v1.0.123/go.mod h1:lv7L70Q/Lc+LARjk6ji+Ihd4Bjb9c/qmqLcdCH8Gzbo= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.2.0 h1:n92B+ZAIzWW3+UFOJpI4ILWZuDcQHHLwVqbWXnZPJdc= -github.com/projectdiscovery/tlsx v1.2.0/go.mod h1:ddPZIluCZBN4N0nn+H0f/QjimF7MOhpQ5RXkJ7zZnwA= +github.com/projectdiscovery/tlsx v1.2.1 h1:R8QgKb/vxd6Y0cfGFBYs4nn0zodHABeeLPqJjs2mNrA= +github.com/projectdiscovery/tlsx v1.2.1/go.mod h1:p19UHGQ6bvcbvhO4NvYBKOxlE4QvrUaectx9g/Mm3JA= github.com/projectdiscovery/useragent v0.0.101 h1:8A+XOJ/nIH+WqW8ogLxJ/psemGp8ATQ2/GuKroJ/81E= github.com/projectdiscovery/useragent v0.0.101/go.mod h1:RGoRw1BQ/lJnhYMbMpEKjyAAgCaDCr/+GsULo5yEJ2I= -github.com/projectdiscovery/utils v0.4.22 h1:OO3FU2uX967sQxu5JtpdBZNzOevvKHAhWqkoTGl+C0A= -github.com/projectdiscovery/utils v0.4.22/go.mod h1:3l84gpCwL9KG1/ZmslOBABCrk84CcpGWJZfR8wZysR4= -github.com/projectdiscovery/wappalyzergo v0.2.38 h1:R7gf5NlndtKd2OFT34uIgFlXDJzE8MXRIwQandAC40M= -github.com/projectdiscovery/wappalyzergo v0.2.38/go.mod h1:XQSgnMQnxliVw2wjKehxeJ4QTRluOJpmm55gLHdztVQ= +github.com/projectdiscovery/utils v0.5.0 h1:DN7mg2DpyObLByuObXzAFEkdNRDoPUnqE5N2szd3b3c= +github.com/projectdiscovery/utils v0.5.0/go.mod h1:eCAWMmyaNxyPWbiKv1oeYJLIKpxceHE2+NWx3Jodhqk= +github.com/projectdiscovery/wappalyzergo v0.2.45 h1:tx0UuYw9GjDy/FMLsL9mr3HjPXoa3qS/lFnda/zQvf4= +github.com/projectdiscovery/wappalyzergo v0.2.45/go.mod h1:1dHfTJRhrbbWBdKwl1p4QKpUDNnPYlHBBL7rEwCDdjM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= @@ -373,6 +372,10 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uRPSpfNZkY= github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E= +github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -386,8 +389,6 @@ github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -400,8 +401,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= @@ -433,12 +434,12 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM= -github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM= -github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA= +github.com/weppos/publicsuffix-go v0.50.0 h1:M178k6l8cnh9T1c1cStkhytVxdk5zPd6gGZf8ySIuVo= +github.com/weppos/publicsuffix-go v0.50.0/go.mod h1:VXhClBYMlDrUsome4pOTpe68Ui0p6iQRAbyHQD1yKoU= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -459,8 +460,8 @@ github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= -github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg= -github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= @@ -505,8 +506,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -539,8 +540,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -571,8 +572,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -594,8 +595,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -636,8 +637,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -650,8 +651,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -667,8 +668,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= @@ -702,8 +703,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index 61327ee..aa6dc48 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -18,7 +18,7 @@ import ( pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/conversion" "github.com/projectdiscovery/utils/env" - errorutil "github.com/projectdiscovery/utils/errors" + "github.com/projectdiscovery/utils/errkit" // unitutils "github.com/projectdiscovery/utils/unit" updateutils "github.com/projectdiscovery/utils/update" urlutil "github.com/projectdiscovery/utils/url" @@ -71,7 +71,7 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential var err error tmp, err := urlutil.Parse(creds.Server) if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("could not parse server url") + return nil, errkit.Wrap(err, "could not parse server url") } tmp.Path = uploadEndpoint tmp.Update() @@ -186,7 +186,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context) { // uploadChunk uploads a chunk of data to the server func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error { if err := u.upload(buff.Bytes()); err != nil { - return errorutil.NewWithErr(err).Msgf("could not upload chunk") + return errkit.Wrap(err, "could not upload chunk") } // if successful, reset the buffer buff.Reset() @@ -198,23 +198,25 @@ func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error { func (u *UploadWriter) upload(data []byte) error { req, err := u.getRequest(data) if err != nil { - return errorutil.NewWithErr(err).Msgf("could not create upload request") + return errkit.Wrap(err, "could not create upload request") } resp, err := u.client.Do(req) if err != nil { - return errorutil.NewWithErr(err).Msgf("could not upload results") + return errkit.Wrap(err, "could not upload results") } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() bin, err := io.ReadAll(resp.Body) if err != nil { - return errorutil.NewWithErr(err).Msgf("could not get id from response") + return errkit.Wrap(err, "could not get id from response") } if resp.StatusCode != http.StatusOK { return fmt.Errorf("could not upload results got status code %v on %v", resp.StatusCode, resp.Request.URL.String()) } var uploadResp uploadResponse if err := json.Unmarshal(bin, &uploadResp); err != nil { - return errorutil.NewWithErr(err).Msgf("could not unmarshal response got %v", string(bin)) + return errkit.Wrapf(err, "could not unmarshal response got %v", string(bin)) } if uploadResp.ID != "" && u.assetGroupID == "" { u.assetGroupID = uploadResp.ID @@ -239,15 +241,15 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { } req, err := retryablehttp.NewRequest(method, url, bytes.NewReader(bin)) if err != nil { - return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request") + return nil, errkit.Wrap(err, "could not create cloud upload request") } // add pdtm meta params - req.URL.Params.Merge(updateutils.GetpdtmParams(runner.Version)) + req.Params.Merge(updateutils.GetpdtmParams(runner.Version)) // if it is upload endpoint also include name if it exists - if u.assetGroupName != "" && req.URL.Path == uploadEndpoint { - req.URL.Params.Add("name", u.assetGroupName) + if u.assetGroupName != "" && req.Path == uploadEndpoint { + req.Params.Add("name", u.assetGroupName) } - req.URL.Update() + req.Update() req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) if u.TeamID != "" { diff --git a/runner/headless.go b/runner/headless.go index 51a24f2..b988583 100644 --- a/runner/headless.go +++ b/runner/headless.go @@ -123,7 +123,9 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim } page = page.Timeout(timeout) - defer page.Close() + defer func() { + _ = page.Close() + }() if err := page.Navigate(url); err != nil { return nil, "", err @@ -150,7 +152,7 @@ func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle tim } func (b *Browser) Close() { - b.engine.Close() - os.RemoveAll(b.tempDir) + _ = b.engine.Close() + _ = os.RemoveAll(b.tempDir) // processutil.CloseProcesses(processutil.IsChromeProcess, b.pids) } diff --git a/runner/healthcheck.go b/runner/healthcheck.go index c48490f..ef79db7 100644 --- a/runner/healthcheck.go +++ b/runner/healthcheck.go @@ -43,7 +43,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { test.WriteString(fmt.Sprintf("Config file \"%s\" Write => %s\n", cfgFilePath, testResult)) c4, err := net.Dial("tcp4", "scanme.sh:80") if err == nil && c4 != nil { - c4.Close() + _ = c4.Close() } testResult = "Ok" if err != nil { @@ -52,7 +52,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { test.WriteString(fmt.Sprintf("IPv4 connectivity to scanme.sh:80 => %s\n", testResult)) c6, err := net.Dial("tcp6", "scanme.sh:80") if err == nil && c6 != nil { - c6.Close() + _ = c6.Close() } testResult = "Ok" if err != nil { diff --git a/runner/options.go b/runner/options.go index bbadb40..1c05397 100644 --- a/runner/options.go +++ b/runner/options.go @@ -31,6 +31,7 @@ import ( fileutil "github.com/projectdiscovery/utils/file" sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" + "github.com/projectdiscovery/utils/structs" updateutils "github.com/projectdiscovery/utils/update" wappalyzer "github.com/projectdiscovery/wappalyzergo" ) @@ -256,11 +257,13 @@ type Options struct { NoFallback bool NoFallbackScheme bool TechDetect bool + CustomFingerprintFile string TLSGrab bool protocol string ShowStatistics bool StatsInterval int RandomAgent bool + AutoReferer bool StoreChain bool StoreVisionReconClusters bool Deny customlist.CustomList @@ -310,6 +313,8 @@ type Options struct { OutputFilterCondition string OutputMatchCondition string StripFilter string + ListOutputFields bool + ExcludeOutputFields goflags.StringSlice //The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err. OnResult OnResultCallback DisableUpdateCheck bool @@ -379,6 +384,7 @@ func ParseOptions() *Options { flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"), flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), + flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"), flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), @@ -432,6 +438,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"), flagSet.DynamicVar(&options.StripFilter, "strip", "html", "strips all tags in response. supported formats: html,xml"), + flagSet.BoolVarP(&options.ListOutputFields, "list-output-fields", "lof", false, "list of fields to output (comma separated)"), + flagSet.StringSliceVarP(&options.ExcludeOutputFields, "exclude-output-fields", "eof", nil, "exclude output fields output based on a condition", goflags.NormalizedOriginalStringSliceOptions), ) flagSet.CreateGroup("rate-limit", "Rate-Limit", @@ -473,7 +481,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.ChainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"), - flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"), + flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11, http2 [experimental], http3 [experimental])"), flagSet.StringVarP(&options.OutputFilterErrorPagePath, "filter-error-page-path", "fepp", "filtered_error_page.json", "path to store filtered error pages"), ) @@ -484,6 +492,7 @@ func ParseOptions() *Options { flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"), + flagSet.BoolVar(&options.AutoReferer, "auto-referer", false, "set the Referer header to the current URL"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), flagSet.StringVarP(&options.Proxy, "proxy", "http-proxy", "", "proxy (http|socks) to use (eg http://127.0.0.1:8080)"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"), @@ -544,6 +553,17 @@ func ParseOptions() *Options { _ = flagSet.Parse() + if options.ListOutputFields { + fields, err := structs.GetStructFields(Result{}) + if err != nil { + gologger.Fatal().Msgf("Could not get struct fields: %s\n", err) + } + for _, field := range fields { + fmt.Println(field) + } + os.Exit(0) + } + if options.OutputAll && options.Output == "" { gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all") } diff --git a/runner/runner.go b/runner/runner.go index 4b47141..1537a44 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -39,8 +39,9 @@ import ( "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" - errorutil "github.com/projectdiscovery/utils/errors" + errorutil "github.com/projectdiscovery/utils/errors" //nolint osutil "github.com/projectdiscovery/utils/os" + "github.com/projectdiscovery/utils/structs" "github.com/Mzack9999/gcache" "github.com/logrusorgru/aurora" @@ -118,15 +119,20 @@ func New(options *Options) (*Runner, error) { if options.Wappalyzer != nil { runner.wappalyzer = options.Wappalyzer } else if options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload { - runner.wappalyzer, err = wappalyzer.New() + runner.wappalyzer, err = func() (*wappalyzer.Wappalyze, error) { + if options.CustomFingerprintFile != "" { + return wappalyzer.NewFromFile(options.CustomFingerprintFile, true, true) + } + return wappalyzer.New() + }() } if err != nil { return nil, errors.Wrap(err, "could not create wappalyzer client") } if options.StoreResponseDir != "" { - os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) - os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) + _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt")) + _ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt")) } httpxOptions := httpx.DefaultOptions @@ -168,6 +174,11 @@ func New(options *Options) (*Runner, error) { } else { httpxOptions.RandomAgent = options.RandomAgent } + if options.CustomHeaders.Has("Referer:") { + httpxOptions.AutoReferer = false + } else { + httpxOptions.AutoReferer = options.AutoReferer + } httpxOptions.ZTLS = options.ZTLS httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave) httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead) @@ -177,6 +188,7 @@ func New(options *Options) (*Runner, error) { } httpxOptions.Resolvers = options.Resolvers httpxOptions.TlsImpersonate = options.TlsImpersonate + httpxOptions.Protocol = httpx.Proto(options.Protocol) var key, value string httpxOptions.CustomHeaders = make(map[string]string) @@ -412,10 +424,27 @@ func (runner *Runner) createNetworkpolicyInstance(options *Options) (*networkpol npOptions.DenyList = append(npOptions.DenyList, exclude) } } + + npOptions.AllowList = appendToList(npOptions.AllowList, options.Allow...) + npOptions.DenyList = appendToList(npOptions.DenyList, options.Deny...) + np, err := networkpolicy.New(npOptions) return np, err } +func appendToList(list []string, values ...string) []string { + for _, value := range values { + switch { + case asn.IsASN(value): + ips := expandASNInputValue(value) + list = append(list, ips...) + default: + list = append(list, value) + } + } + return list +} + func expandCIDRInputValue(value string) []string { var ips []string ipsCh, _ := mapcidr.IPAddressesAsStream(value) @@ -673,7 +702,7 @@ func makePrintCallback() func(stats clistats.StatisticsClient) interface{} { builder.WriteString(clistats.String(uint64(incrementRequests / duration.Seconds()))) builder.WriteString(" | Requests: ") - builder.WriteString(fmt.Sprintf("%.0f", currentRequests)) + _, _ = fmt.Fprintf(builder, "%.0f", currentRequests) hosts, _ := stats.GetCounter("hosts") totalHosts, _ := stats.GetStatic("totalHosts") @@ -784,18 +813,26 @@ func (r *Runner) RunEnumeration() { if r.options.Output != "" && r.options.OutputAll { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) - defer plainFile.Close() + defer func() { + _ = plainFile.Close() + }() jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json") - defer jsonFile.Close() + defer func() { + _ = jsonFile.Close() + }() csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv") - defer csvFile.Close() + defer func() { + _ = csvFile.Close() + }() } jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput) jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput) if r.options.Output != "" && plainFile == nil && !jsonOrCsv { plainFile = openOrCreateFile(r.options.Resume, r.options.Output) - defer plainFile.Close() + defer func() { + _ = plainFile.Close() + }() } if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil { @@ -804,7 +841,9 @@ func (r *Runner) RunEnumeration() { ext = ".json" } jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) - defer jsonFile.Close() + defer func() { + _ = jsonFile.Close() + }() } if r.options.Output != "" && r.options.CSVOutput && csvFile == nil { @@ -813,7 +852,9 @@ func (r *Runner) RunEnumeration() { ext = ".csv" } csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) - defer csvFile.Close() + defer func() { + _ = csvFile.Close() + }() } if r.options.CSVOutput { @@ -1076,7 +1117,7 @@ func (r *Runner) RunEnumeration() { // store response if r.scanopts.StoreResponse || r.scanopts.StoreChain { if r.scanopts.OmitBody { - resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1) + resp.Raw = strings.ReplaceAll(resp.Raw, resp.ResponseBody, "") } responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) @@ -1153,6 +1194,12 @@ func (r *Runner) RunEnumeration() { plainFile.WriteString(resp.str + "\n") } + if len(r.options.ExcludeOutputFields) > 0 { + if filteredData, err := structs.FilterStruct(resp, nil, r.options.ExcludeOutputFields); err == nil { + resp = filteredData + } + } + // call the callback function if any // be careful and check for result.Err if r.options.OnResult != nil { @@ -1204,7 +1251,9 @@ func (r *Runner) RunEnumeration() { if err != nil { gologger.Warning().Msgf("Could not create HTML file %s\n", err) } - defer screenshotHtml.Close() + defer func() { + _ = screenshotHtml.Close() + }() templateMap := template.FuncMap{ "safeURL": func(u string) template.URL { @@ -1296,7 +1345,9 @@ func (r *Runner) RunEnumeration() { if err != nil { gologger.Fatal().Msgf("Failed to create JSON file: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() _, err = file.Write(clusterReportJSON) if err != nil { @@ -1320,7 +1371,9 @@ func logFilteredErrorPage(fileName, url string) { gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", fileName, err) return } - defer file.Close() + defer func() { + _ = file.Close() + }() info := map[string]interface{}{ "url": url, @@ -1621,7 +1674,7 @@ retry: var requestDump []byte if scanopts.Unsafe { var errDump error - requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) + requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) if errDump != nil { return Result{URL: URL.String(), Input: origInput, Err: errDump} } @@ -1644,7 +1697,7 @@ retry: } } // fix the final output url - fullURL := req.URL.String() + fullURL := req.String() if parsedURL, errParse := r.parseURL(fullURL); errParse != nil { return Result{URL: URL.String(), Input: origInput, Err: errParse} } else { @@ -1830,7 +1883,7 @@ retry: serverHeader := resp.GetHeader("Server") if scanopts.OutputServerHeader { - builder.WriteString(fmt.Sprintf(" [%s]", serverHeader)) + _, _ = fmt.Fprintf(builder, " [%s]", serverHeader) } var ( @@ -1944,7 +1997,7 @@ retry: } if scanopts.OutputIP || scanopts.ProbeAllIPS { - builder.WriteString(fmt.Sprintf(" [%s]", ip)) + _, _ = fmt.Fprintf(builder, " [%s]", ip) } var onlyHost string @@ -1969,16 +2022,16 @@ retry: if scanopts.OutputCName && len(cnames) > 0 { // Print only the first CNAME (full list in json) - builder.WriteString(fmt.Sprintf(" [%s]", cnames[0])) + _, _ = fmt.Fprintf(builder, " [%s]", cnames[0]) } isCDN, cdnName, cdnType, err := hp.CdnCheck(ip) if scanopts.OutputCDN == "true" && isCDN && err == nil { - builder.WriteString(fmt.Sprintf(" [%s]", cdnName)) + _, _ = fmt.Fprintf(builder, " [%s]", cdnName) } if scanopts.OutputResponseTime { - builder.WriteString(fmt.Sprintf(" [%s]", resp.Duration)) + _, _ = fmt.Fprintf(builder, " [%s]", resp.Duration) } technologyDetails := make(map[string]wappalyzer.AppInfo) @@ -2042,7 +2095,7 @@ retry: hashesMap := make(map[string]interface{}) if scanopts.Hashes != "" { hs := strings.Split(scanopts.Hashes, ",") - outputHashes := !(r.options.JSONOutput || r.options.OutputAll) + outputHashes := !(r.options.JSONOutput || r.options.OutputAll) //nolint if outputHashes { builder.WriteString(" [") } @@ -2095,7 +2148,7 @@ retry: if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(resp.Lines).String()) } else { - builder.WriteString(fmt.Sprint(resp.Lines)) + _, _ = fmt.Fprintf(builder, "%d", resp.Lines) } builder.WriteRune(']') } @@ -2106,7 +2159,7 @@ retry: if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) } else { - builder.WriteString(fmt.Sprint(jarmhash)) + _, _ = fmt.Fprintf(builder, "%s", jarmhash) } builder.WriteRune(']') } @@ -2115,7 +2168,7 @@ retry: if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(resp.Words).String()) } else { - builder.WriteString(fmt.Sprint(resp.Words)) + _, _ = fmt.Fprintf(builder, "%d", resp.Words) } builder.WriteRune(']') } @@ -2133,7 +2186,7 @@ retry: // store response if scanopts.StoreResponse || scanopts.StoreChain { if r.options.OmitBody { - resp.Raw = strings.Replace(resp.Raw, string(resp.Data), "", -1) + resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "") } responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) // URL.EscapedString returns that can be used as filename @@ -2336,9 +2389,9 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) { func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, finalURL string, defaultProbe bool) (string, string, string, []byte, string, error) { // Check if current URI is ending with .ico => use current body without additional requests - if path.Ext(req.URL.Path) == ".ico" { + if path.Ext(req.Path) == ".ico" { mmh3, md5h, err := r.calculateFaviconHashWithRaw(currentResp) - return mmh3, md5h, req.URL.Path, currentResp, req.URL.String(), err + return mmh3, md5h, req.Path, currentResp, req.String(), err } // Parse HTML: collect hrefs + optional @@ -2353,7 +2406,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } // Determine base URL: prefer finalURL (redirect target) then apply - baseNet, _ := url.Parse(req.URL.String()) + baseNet, _ := url.Parse(req.String()) if finalURL != "" { if u, err := url.Parse(finalURL); err == nil { baseNet = u @@ -2441,7 +2494,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, faviconMMH3 = mmh3 faviconMD5 = md5h faviconPath = raw - faviconURL = clone.URL.String() + faviconURL = clone.String() faviconData = respFav.Data gologger.Debug().Msgf("favicon resolved url=%s raw_href=%s size=%d bytes", faviconURL, faviconPath, len(faviconData)) break @@ -2453,7 +2506,7 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, string, error) { hashNum, md5Hash, err := stringz.FaviconHash(data) if err != nil { - return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) + return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) //nolint } return fmt.Sprintf("%d", hashNum), md5Hash, nil } diff --git a/runner/runner_test.go b/runner/runner_test.go index f88e012..850566b 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -222,3 +222,93 @@ func TestRunner_CSVRow(t *testing.T) { t.Error("CSV sanitization incorrectly modified non-vulnerable field") } } + +func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) { + runner := &Runner{} + + tests := []struct { + name string + allow []string + deny []string + testCases []struct { + ip string + expected bool + reason string + } + }{ + { + name: "Allow flag blocks IPs outside allowed range", + allow: []string{"192.168.1.0/24"}, + deny: nil, + testCases: []struct { + ip string + expected bool + reason string + }{ + {"8.8.8.8", false, "IP outside allowed range should be blocked"}, + {"192.168.1.10", true, "IP inside allowed range should be allowed"}, + }, + }, + { + name: "Deny flag blocks IPs in denied range", + allow: nil, + deny: []string{"127.0.0.0/8"}, + testCases: []struct { + ip string + expected bool + reason string + }{ + {"127.0.0.1", false, "IP in denied range should be blocked"}, + {"8.8.8.8", true, "IP outside denied range should be allowed"}, + }, + }, + { + name: "Combined Allow and Deny flags", + allow: []string{"192.168.0.0/16"}, + deny: []string{"192.168.1.0/24"}, + testCases: []struct { + ip string + expected bool + reason string + }{ + {"10.0.0.1", false, "IP outside allowed range should be blocked"}, + {"192.168.1.100", false, "IP in denied range should be blocked even if in allowed range"}, + {"192.168.2.50", true, "IP in allowed range but not in denied range should be allowed"}, + }, + }, + { + name: "Multiple Allow and Deny ranges", + allow: []string{"10.0.0.0/8", "172.16.0.0/12"}, + deny: []string{"10.1.0.0/16", "172.20.0.0/16"}, + testCases: []struct { + ip string + expected bool + reason string + }{ + {"10.0.1.1", true, "10.0.1.1 should be allowed (in allow range, not in deny)"}, + {"10.1.1.1", false, "10.1.1.1 should be blocked (in deny range)"}, + {"172.16.1.1", true, "172.16.1.1 should be allowed (in allow range, not in deny)"}, + {"172.20.1.1", false, "172.20.1.1 should be blocked (in deny range)"}, + {"192.168.1.1", false, "192.168.1.1 should be blocked (not in any allow range)"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + options := &Options{ + Allow: tc.allow, + Deny: tc.deny, + } + + np, err := runner.createNetworkpolicyInstance(options) + require.Nil(t, err, "could not create networkpolicy instance") + require.NotNil(t, np, "networkpolicy instance should not be nil") + + for _, testCase := range tc.testCases { + allowed := np.Validate(testCase.ip) + require.Equal(t, testCase.expected, allowed, testCase.reason) + } + }) + } +} diff --git a/snapcraft.yaml b/snapcraft.yaml index f15a9b6..9f4b4f0 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -2,7 +2,7 @@ name: httpx summary: httpx is a fast and multi-purpose HTTP toolkit description: | httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads. -version: 'v1.2.1' +version: 'v1.7.1' icon: static/httpx-logo.png license: MIT base: core18