admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

restoring original logic + limiting read to 512Mb + lint

Browse Source
This commit is contained in:
Mzack9999
2025-12-24 15:00:10 +04:00
parent 85695760cb
commit 78bf9c393a
8 changed files with 50 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -3
View File
@@ -254,8 +254,8 @@ OPTIMIZATIONS:
-retries int number of retries
-timeout int timeout in seconds (default 10)
-delay value duration between each http request (eg: 200ms, 1s) (default -1ns)
-rsts, -response-size-to-save int max response size to save in bytes (default 10485760)
-rstr, -response-size-to-read int max response size to read in bytes (default 10485760)
-rsts, -response-size-to-save int max response size to save in bytes (default 2147483647)
-rstr, -response-size-to-read int max response size to read in bytes (default 2147483647)
CLOUD:
-auth configure projectdiscovery cloud (pdcp) api key (default true)
@@ -307,4 +307,4 @@ Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomno
<a href="https://discord.gg/projectdiscovery"><img src="https://raw.githubusercontent.com/projectdiscovery/nuclei-burp-plugin/main/static/join-discord.png" width="300" alt="Join Discord"></a>
</div>
</div>
cmd/httpx/resume.cfg
+2
View File
@@ -0,0 +1,2 @@
resume_from=http://localhost:8000/endless
index=1
common/httpx/httpx.go
+22 -25
View File
@@ -7,7 +7,6 @@ import (
"io"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"strconv"
@@ -236,49 +235,47 @@ get_response:
resp.Headers = httpresp.Header.Clone()
// Dump headers only (does not consume body)
headers, err := httputil.DumpResponse(httpresp, false)
if h.Options.MaxResponseBodySizeToRead > 0 {
httpresp.Body = io.NopCloser(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
defer func() {
_, _ = io.Copy(io.Discard, httpresp.Body)
_ = httpresp.Body.Close()
}()
}
// httputil.DumpResponse does not handle websockets
headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp)
if err != nil {
if stringsutil.ContainsAny(err.Error(), "tls: user canceled") {
shouldIgnoreErrors = true
shouldIgnoreBodyErrors = true
}
// Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read.
// The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header
if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") {
gzipRetry = true
req.Header.Set("Accept-Encoding", "identity")
goto get_response
}
if !shouldIgnoreErrors {
return nil, err
}
}
resp.Raw = string(rawResp)
resp.RawHeaders = string(headers)
var respbody []byte
// body shouldn't be read with the following status codes
// 101 - Switching Protocols => websockets don't have a readable body
// 304 - Not Modified => no body the response terminates with latest header newline
if !generic.EqualsAny(httpresp.StatusCode, http.StatusSwitchingProtocols, http.StatusNotModified) {
var err error
respbody, err = io.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
if err != nil {
// Edge case: some servers respond with gzip encoding header but uncompressed body.
// Retry request with identity encoding.
if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") {
gzipRetry = true
req.Header.Set("Accept-Encoding", "identity")
goto get_response
}
if !shouldIgnoreBodyErrors {
return nil, err
}
if err != nil && !shouldIgnoreBodyErrors {
return nil, err
}
}
// Build bounded raw response: headers + capped body
// NOTE: resp.Raw must be constructed from a capped body to avoid OOM on infinite streams.
raw := make([]byte, 0, len(headers)+len(respbody))
raw = append(raw, headers...)
raw = append(raw, respbody...)
resp.Raw = string(raw)
closeErr := httpresp.Body.Close()
if closeErr != nil && !shouldIgnoreBodyErrors {
return nil, closeErr
common/httpx/option.go
+10 -1
View File
@@ -5,10 +5,19 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize"
"github.com/projectdiscovery/cdncheck"
"github.com/projectdiscovery/networkpolicy"
)
// DefaultMaxResponseBodySize is the default maximum response body size (4GB)
var DefaultMaxResponseBodySize int64
func init() {
maxResponseBodySize, _ := humanize.ParseBytes("512Mb")
DefaultMaxResponseBodySize = int64(maxResponseBodySize)
}
// Options contains configuration options for the client
type Options struct {
RandomAgent bool
@@ -66,7 +75,7 @@ var DefaultOptions = Options{
Unsafe: false,
CdnCheck: "true",
ExcludeCdn: false,
MaxResponseBodySizeToRead: 1024 * 1024 * 10,
MaxResponseBodySizeToRead: DefaultMaxResponseBodySize,
// VHOSTs options
VHostIgnoreStatusCode: false,
VHostIgnoreContentLength: true,
common/stringz/stringz.go
+3 -2
View File
@@ -85,9 +85,10 @@ func AddURLDefaultPort(rawURL string) string {
}
// Force default port to be added if not present
if u.Port() == "" {
if u.Scheme == urlutil.HTTP {
switch u.Scheme {
case urlutil.HTTP:
u.UpdatePort("80")
} else if u.Scheme == urlutil.HTTPS {
case urlutil.HTTPS:
u.UpdatePort("443")
}
}
go.mod
+1
View File
@@ -51,6 +51,7 @@ require (
require (
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0
github.com/dustin/go-humanize v1.0.1
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
github.com/weppos/publicsuffix-go v0.50.1
go.sum
+2
View File
@@ -116,6 +116,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
runner/options.go
+7 -5
View File
@@ -22,7 +22,7 @@ import (
"github.com/projectdiscovery/httpx/common/customlist"
customport "github.com/projectdiscovery/httpx/common/customports"
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
"github.com/projectdiscovery/httpx/common/httpx"
httpxcommon "github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/networkpolicy"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -540,9 +540,11 @@ func ParseOptions() *Options {
flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),
// 10MB max response size matches common/httpx default
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", (10*1024*1024), "max response size to save in bytes"),
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", (10*1024*1024), "max response size to read in bytes"),
)
flagSet.CreateGroup("response", "Response",
flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to save in bytes"),
flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", int(httpxcommon.DefaultMaxResponseBodySize), "max response size to read in bytes"),
)
flagSet.CreateGroup("cloud", "Cloud",
@@ -772,7 +774,7 @@ func (options *Options) ValidateOptions() error {
options.OutputCDN = "true"
}
if !stringsutil.EqualFoldAny(options.Protocol, string(httpx.UNKNOWN), string(httpx.HTTP11)) {
if !stringsutil.EqualFoldAny(options.Protocol, string(httpxcommon.UNKNOWN), string(httpxcommon.HTTP11)) {
return fmt.Errorf("invalid protocol: %s", options.Protocol)
}