From c48909f64f8250e585ea7e7ea24891660fc82387 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 17 Aug 2022 02:16:06 +0300 Subject: [PATCH] Minor fixes and improvements (#726) * Added .idea to .gitignore * Updated the README * Added more verbose information on preset extractors * removing deprecated ioutil Co-authored-by: Mzack9999 --- .gitignore | 4 +-- README.md | 36 +++++++++++++-------------- cmd/integration-test/http.go | 6 ++--- common/customextract/customextract.go | 2 +- common/fileutil/fileutil.go | 4 +-- common/httputilz/httputilz.go | 4 +-- common/httpx/encodings.go | 8 +++--- common/httpx/http2.go | 3 +-- common/httpx/httpx.go | 3 +-- go.mod | 7 +++--- go.sum | 10 +++++--- runner/options.go | 16 ++++++------ runner/runner.go | 23 ++++++++++------- 13 files changed, 67 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index b8f8c16..028e9b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ - +.idea/ +.vscode/ cmd/httpx/httpx integration_tests/httpx integration_tests/integration-test @@ -6,4 +7,3 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg -.vscode/ diff --git a/README.md b/README.md index 77f2e4a..1ada6ee 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@

-httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library, it is designed to maintain the result reliability with increased threads. +httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads. # Features @@ -42,23 +42,23 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi ### Supported probes:- -| Probes | Default check | Probes | Default check | -| --------------- | ------------- | ----------------- | ------------- | -| URL | true | IP | true | -| Title | true | CNAME | true | -| Status Code | true | Raw HTTP | false | -| Content Length | true | HTTP2 | false | -| TLS Certificate | true | HTTP Pipeline | false | -| CSP Header | true | Virtual host | false | -| Line Count | true | Word Count | true | -| Location Header | true | CDN | false | -| Web Server | true | Paths | false | -| Web Socket | true | Ports | false | -| Response Time | true | Request Method | true | -| Favicon Hash | false | Probe Status | false | -| Body Hash | true | Header Hash | true | -| Redirect chain | false | URL Scheme | true | -| JARM Hash | false | ASN | false | +| Probes | Default check | Probes | Default check | +|-----------------|---------------|----------------|---------------| +| URL | true | IP | true | +| Title | true | CNAME | true | +| Status Code | true | Raw HTTP | false | +| Content Length | true | HTTP2 | false | +| TLS Certificate | true | HTTP Pipeline | false | +| CSP Header | true | Virtual host | false | +| Line Count | true | Word Count | true | +| Location Header | true | CDN | false | +| Web Server | true | Paths | false | +| Web Socket | true | Ports | false | +| Response Time | true | Request Method | true | +| Favicon Hash | false | Probe Status | false | +| Body Hash | true | Header Hash | true | +| Redirect chain | false | URL Scheme | true | +| JARM Hash | false | ASN | false | # Installation Instructions diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index bd0f6d8..72fd687 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" @@ -207,7 +207,7 @@ func (h *issue400) Execute() error { router := httprouter.New() router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") - data, _ := ioutil.ReadAll(r.Body) + data, _ := io.ReadAll(r.Body) fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) @@ -231,7 +231,7 @@ func (h *issue414) Execute() error { router := httprouter.New() router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") - data, _ := ioutil.ReadAll(r.Body) + data, _ := io.ReadAll(r.Body) fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) diff --git a/common/customextract/customextract.go b/common/customextract/customextract.go index d15afca..b3966d7 100644 --- a/common/customextract/customextract.go +++ b/common/customextract/customextract.go @@ -4,6 +4,6 @@ import "regexp" var ExtractPresets = map[string]*regexp.Regexp{ "url": regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint - "ip": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint + "ipv4": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint "mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"), //nolint } diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 8fe5c6e..76213cb 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -3,7 +3,7 @@ package fileutil import ( "bufio" "errors" - "io/ioutil" + "net" "os" "path/filepath" @@ -72,7 +72,7 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m } else if _, _, err := net.ParseCIDR(item); err == nil { networkList = append(networkList, item) } else if fileutil.FileExists(item) { - if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 { + if filedata, err := os.ReadFile(item); err == nil && len(filedata) > 0 { networkList = append(networkList, LoadCidrsFromSliceOrFileWithMaxRecursion(string(filedata), "\n", maxRecursion-1)...) } } diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go index 8b6367c..1aaa6fc 100644 --- a/common/httputilz/httputilz.go +++ b/common/httputilz/httputilz.go @@ -3,7 +3,7 @@ package httputilz import ( "bufio" "fmt" - "io/ioutil" + "io" "net/http/httputil" "strings" @@ -87,7 +87,7 @@ func ParseRequest(req string, unsafe bool) (method, path string, headers map[str } // Set the request body - b, err := ioutil.ReadAll(reader) + b, err := io.ReadAll(reader) if err != nil { err = fmt.Errorf("could not read request body: %s", err) return diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index 752e2e8..9c5114b 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -2,7 +2,7 @@ package httpx import ( "bytes" - "io/ioutil" + "io" "net/http" "strings" @@ -19,7 +19,7 @@ import ( func Decodegbk(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } @@ -30,7 +30,7 @@ func Decodegbk(s []byte) ([]byte, error) { func Decodebig5(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } @@ -41,7 +41,7 @@ func Decodebig5(s []byte) ([]byte, error) { func Encodebig5(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } diff --git a/common/httpx/http2.go b/common/httpx/http2.go index a1f0bb3..9d1a705 100644 --- a/common/httpx/http2.go +++ b/common/httpx/http2.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "github.com/projectdiscovery/gologger" @@ -68,7 +67,7 @@ func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool { } func freeHTTPResources(response *http.Response) error { - _, err := io.Copy(ioutil.Discard, response.Body) + _, err := io.Copy(io.Discard, response.Body) if err != nil { return fmt.Errorf("could not discard response body: %s", err) } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 4438e6f..c524ade 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -202,7 +201,7 @@ get_response: // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error - respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) + respbody, err = io.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) if err != nil && !shouldIgnoreBodyErrors { return nil, err } diff --git a/go.mod b/go.mod index a21ff10..159b7ee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.17 +go 1.18 require ( github.com/akrylysov/pogreb v0.10.1 // indirect @@ -51,6 +51,7 @@ require ( github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 go.uber.org/multierr v1.8.0 + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e ) require ( @@ -77,8 +78,8 @@ require ( github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index a63f3b0..5fe1ae7 100644 --- a/go.sum +++ b/go.sum @@ -140,7 +140,6 @@ github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= -github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -254,10 +253,13 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -276,7 +278,6 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -327,8 +328,9 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/runner/options.go b/runner/options.go index 91252d7..920cf13 100644 --- a/runner/options.go +++ b/runner/options.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/pkg/errors" + "golang.org/x/exp/maps" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" @@ -16,6 +17,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/customheader" "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" @@ -251,7 +253,7 @@ func ParseOptions() *Options { options := &Options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.`) flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), @@ -294,8 +296,8 @@ func ParseOptions() *Options { ) flagSet.CreateGroup("extractor", "Extractor", - flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex", goflags.StringSliceOptions), - flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "display response content with matched regex", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, fmt.Sprintf("display response content matched by a pre-defined regex (%s)", strings.Join(maps.Keys(customextract.ExtractPresets), ",")), goflags.StringSliceOptions), ) flagSet.CreateGroup("filters", "Filters", @@ -343,8 +345,8 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), - flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "Custom TLS SNI name"), - flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), + flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"), @@ -419,11 +421,11 @@ func ParseOptions() *Options { func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - return fmt.Errorf("File %s does not exist.", options.InputFile) + return fmt.Errorf("File '%s' does not exist.", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - return fmt.Errorf("File %s does not exist.", options.InputRawRequest) + return fmt.Errorf("File '%s' does not exist.", options.InputRawRequest) } multiOutput := options.CSVOutput && options.JSONOutput diff --git a/runner/runner.go b/runner/runner.go index 6613cca..26bbbb6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -7,7 +7,7 @@ import ( "encoding/csv" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" "net/http/httputil" @@ -22,6 +22,8 @@ import ( "strings" "time" + "golang.org/x/exp/maps" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -30,6 +32,7 @@ import ( "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" @@ -39,6 +42,9 @@ import ( "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/ratelimit" + // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/fileutil" @@ -55,8 +61,6 @@ import ( "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/rawhttp" wappalyzer "github.com/projectdiscovery/wappalyzergo" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/ratelimit" ) // Runner is a client for running the enumeration process. @@ -155,7 +159,7 @@ func New(options *Options) (*Runner, error) { if options.InputRawRequest != "" { var rawRequest []byte - rawRequest, err = ioutil.ReadFile(options.InputRawRequest) + rawRequest, err = os.ReadFile(options.InputRawRequest) if err != nil { gologger.Fatal().Msgf("Could not read raw request from path '%s': %s\n", options.InputRawRequest, err) } @@ -248,7 +252,8 @@ func New(options *Options) (*Runner, error) { if regex, ok := customextract.ExtractPresets[regexName]; ok { scanopts.extractRegexps[regexName] = regex } else { - gologger.Warning().Msgf("Could not find preset: %s\n", regexName) + availablePresets := strings.Join(maps.Keys(customextract.ExtractPresets), ",") + gologger.Warning().Msgf("Could not find preset: '%s'. Available presets are: %s\n", regexName, availablePresets) } } } @@ -983,7 +988,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -1010,7 +1015,7 @@ retry: // Create a copy on the fly of the request body if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) @@ -1455,14 +1460,14 @@ retry: if len(respRaw) > scanopts.MaxResponseBodySizeToSave { respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] } - writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) + writeErr := os.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) - writeErr := ioutil.WriteFile(responsePath, []byte(resp.GetChain()), 0644) + writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) }