From 78bf9c393a04aa37bba3f317f2e47859f2e8dcaf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 24 Dec 2025 15:00:10 +0400 Subject: [PATCH] restoring original logic + limiting read to 512Mb + lint --- README.md | 6 ++--- cmd/httpx/resume.cfg | 2 ++ common/httpx/httpx.go | 47 ++++++++++++++++++--------------------- common/httpx/option.go | 11 ++++++++- common/stringz/stringz.go | 5 +++-- go.mod | 1 + go.sum | 2 ++ runner/options.go | 12 +++++----- 8 files changed, 50 insertions(+), 36 deletions(-) create mode 100644 cmd/httpx/resume.cfg diff --git a/README.md b/README.md index 4d11270..978b4a1 100644 --- a/README.md +++ b/README.md @@ -254,8 +254,8 @@ OPTIMIZATIONS: -retries int number of retries -timeout int timeout in seconds (default 10) -delay value duration between each http request (eg: 200ms, 1s) (default -1ns) - -rsts, -response-size-to-save int max response size to save in bytes (default 10485760) - -rstr, -response-size-to-read int max response size to read in bytes (default 10485760) + -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) + -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) CLOUD: -auth configure projectdiscovery cloud (pdcp) api key (default true) @@ -307,4 +307,4 @@ Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomno Join Discord - + \ No newline at end of file diff --git a/cmd/httpx/resume.cfg b/cmd/httpx/resume.cfg new file mode 100644 index 0000000..7cb2a56 --- /dev/null +++ b/cmd/httpx/resume.cfg @@ -0,0 +1,2 @@ +resume_from=http://localhost:8000/endless +index=1 diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index d707052..039f4c4 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -7,7 +7,6 @@ import ( "io" "net" "net/http" - "net/http/httputil" "net/url" "os" "strconv" @@ -236,49 +235,47 @@ get_response: resp.Headers = httpresp.Header.Clone() - // Dump headers only (does not consume body) - headers, err := httputil.DumpResponse(httpresp, false) + if h.Options.MaxResponseBodySizeToRead > 0 { + httpresp.Body = io.NopCloser(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) + defer func() { + _, _ = io.Copy(io.Discard, httpresp.Body) + _ = httpresp.Body.Close() + }() + } + + // httputil.DumpResponse does not handle websockets + headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp) if err != nil { if stringsutil.ContainsAny(err.Error(), "tls: user canceled") { shouldIgnoreErrors = true shouldIgnoreBodyErrors = true } + + // Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read. + // The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header + if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") { + gzipRetry = true + req.Header.Set("Accept-Encoding", "identity") + goto get_response + } if !shouldIgnoreErrors { return nil, err } } - + resp.Raw = string(rawResp) resp.RawHeaders = string(headers) - var respbody []byte // body shouldn't be read with the following status codes // 101 - Switching Protocols => websockets don't have a readable body // 304 - Not Modified => no body the response terminates with latest header newline if !generic.EqualsAny(httpresp.StatusCode, http.StatusSwitchingProtocols, http.StatusNotModified) { - + var err error respbody, err = io.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) - if err != nil { - // Edge case: some servers respond with gzip encoding header but uncompressed body. - // Retry request with identity encoding. - if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") { - gzipRetry = true - req.Header.Set("Accept-Encoding", "identity") - goto get_response - } - if !shouldIgnoreBodyErrors { - return nil, err - } + if err != nil && !shouldIgnoreBodyErrors { + return nil, err } } - // Build bounded raw response: headers + capped body - // NOTE: resp.Raw must be constructed from a capped body to avoid OOM on infinite streams. - - raw := make([]byte, 0, len(headers)+len(respbody)) - raw = append(raw, headers...) - raw = append(raw, respbody...) - resp.Raw = string(raw) - closeErr := httpresp.Body.Close() if closeErr != nil && !shouldIgnoreBodyErrors { return nil, closeErr diff --git a/common/httpx/option.go b/common/httpx/option.go index b64cfd3..8e57a32 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -5,10 +5,19 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/networkpolicy" ) +// DefaultMaxResponseBodySize is the default maximum response body size (4GB) +var DefaultMaxResponseBodySize int64 + +func init() { + maxResponseBodySize, _ := humanize.ParseBytes("512Mb") + DefaultMaxResponseBodySize = int64(maxResponseBodySize) +} + // Options contains configuration options for the client type Options struct { RandomAgent bool @@ -66,7 +75,7 @@ var DefaultOptions = Options{ Unsafe: false, CdnCheck: "true", ExcludeCdn: false, - MaxResponseBodySizeToRead: 1024 * 1024 * 10, + MaxResponseBodySizeToRead: DefaultMaxResponseBodySize, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 5033e1c..294dc9d 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -85,9 +85,10 @@ func AddURLDefaultPort(rawURL string) string { } // Force default port to be added if not present if u.Port() == "" { - if u.Scheme == urlutil.HTTP { + switch u.Scheme { + case urlutil.HTTP: u.UpdatePort("80") - } else if u.Scheme == urlutil.HTTPS { + case urlutil.HTTPS: u.UpdatePort("443") } } diff --git a/go.mod b/go.mod index cdf1d1a..3cde4b0 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 + github.com/dustin/go-humanize v1.0.1 github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/weppos/publicsuffix-go v0.50.1 diff --git a/go.sum b/go.sum index 6471e0b..ba9a432 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= diff --git a/runner/options.go b/runner/options.go index 5936c37..7c74ff8 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,7 +22,7 @@ import ( "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" - "github.com/projectdiscovery/httpx/common/httpx" + httpxcommon "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/networkpolicy" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" @@ -540,9 +540,11 @@ func ParseOptions() *Options { flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), - // 10MB max response size matches common/httpx default - flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", (10*1024*1024), "max response size to save in bytes"), - flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", (10*1024*1024), "max response size to read in bytes"), + ) + + flagSet.CreateGroup("response", "Response", + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to read in bytes"), ) flagSet.CreateGroup("cloud", "Cloud", @@ -772,7 +774,7 @@ func (options *Options) ValidateOptions() error { options.OutputCDN = "true" } - if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) { + if !stringsutil.EqualFoldAny(options.Protocol, string(httpxcommon.UNKNOWN), string(httpxcommon.HTTP11)) { return fmt.Errorf("invalid protocol: %s", options.Protocol) }