From d4e72d805b4785f79f02ee7acc34d9aeb1702189 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 2 Oct 2020 22:39:37 +0200 Subject: [PATCH 1/3] making cdn check optional --- cmd/httpx/httpx.go | 7 ++++--- common/httpx/cdn.go | 13 +++++++++---- common/httpx/httpx.go | 8 +++++--- common/httpx/option.go | 3 ++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index b59e7c0..26b9959 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -47,6 +47,7 @@ func main() { httpxOptions.HTTPProxy = options.HTTPProxy httpxOptions.Unsafe = options.Unsafe httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI} + httpxOptions.CdnCheck = options.OutputCDN var key, value string httpxOptions.CustomHeaders = make(map[string]string) @@ -542,8 +543,8 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", cnames[0])) } - isCDN := hp.CdnCheck(ip) - if scanopts.OutputCDN && isCDN { + isCDN, err := hp.CdnCheck(ip) + if scanopts.OutputCDN && isCDN && err == nil { builder.WriteString(" [cdn]") } @@ -621,7 +622,7 @@ type Result struct { WebSocket bool `json:"websocket,omitempty"` Pipeline bool `json:"pipeline,omitempty"` HTTP2 bool `json:"http2"` - CDN bool `json:"cdn"` + CDN bool `json:"cdn,omitempty"` Duration time.Duration `json:"duration"` } diff --git a/common/httpx/cdn.go b/common/httpx/cdn.go index 942fee0..7bbf276 100644 --- a/common/httpx/cdn.go +++ b/common/httpx/cdn.go @@ -1,10 +1,15 @@ package httpx -import "net" +import ( + "fmt" + "net" +) // CdnCheck verifies if the given ip is part of Cdn ranges -func (h *HTTPX) CdnCheck(ip string) bool { - ok, err := h.cdn.Check(net.ParseIP((ip))) +func (h *HTTPX) CdnCheck(ip string) (bool, error) { + if h.cdn == nil { + return false, fmt.Errorf("cdn client not configured") + } - return ok && err == nil + return h.cdn.Check(net.ParseIP((ip))) } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index bf4729d..eb884f9 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -103,9 +103,11 @@ func New(options *Options) (*HTTPX, error) { httpx.htmlPolicy = bluemonday.NewPolicy() httpx.CustomHeaders = httpx.Options.CustomHeaders httpx.RequestOverride = &options.RequestOverride - httpx.cdn, err = cdncheck.New() - if err != nil { - return nil, fmt.Errorf("could not create cdn check: %s", err) + if options.CdnCheck { + httpx.cdn, err = cdncheck.New() + if err != nil { + return nil, fmt.Errorf("could not create cdn check: %s", err) + } } return httpx, nil diff --git a/common/httpx/option.go b/common/httpx/option.go index c7bb2d8..256b121 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -11,6 +11,7 @@ type Options struct { HTTPProxy string SocksProxy string Threads int + CdnCheck bool // Timeout is the maximum time to wait for the request Timeout time.Duration // RetryMax is the maximum number of retries @@ -35,6 +36,7 @@ var DefaultOptions = Options{ Timeout: 30 * time.Second, RetryMax: 5, Unsafe: false, + CdnCheck: true, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, @@ -43,5 +45,4 @@ var DefaultOptions = Options{ VHostStripHTML: false, VHostSimilarityRatio: 85, DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)", - // Smuggling Options } From 1a5a4da9a6a2bfa17970b1f55100ff4b557cb368 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 3 Oct 2020 20:36:56 +0200 Subject: [PATCH 2/3] adding optional cdncheck with pd cdn endpoint to avoid users' ip blocking --- common/httpx/httpx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index eb884f9..b4f69d7 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -104,7 +104,7 @@ func New(options *Options) (*HTTPX, error) { httpx.CustomHeaders = httpx.Options.CustomHeaders httpx.RequestOverride = &options.RequestOverride if options.CdnCheck { - httpx.cdn, err = cdncheck.New() + httpx.cdn, err = cdncheck.NewWithCache() if err != nil { return nil, fmt.Errorf("could not create cdn check: %s", err) } From 785b2b2b6514388509a4dd0251ad1a7fe9c6da9c Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 3 Oct 2020 20:44:29 +0200 Subject: [PATCH 3/3] updated go modules --- go.mod | 4 ++-- go.sum | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index f869650..088235f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.4 github.com/miekg/dns v1.1.31 - github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26 + github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 github.com/projectdiscovery/fdmax v0.0.2 github.com/projectdiscovery/gologger v1.0.1 github.com/projectdiscovery/mapcidr v0.0.4 @@ -16,6 +16,6 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.2.1 - golang.org/x/net v0.0.0-20200927032502-5d4f70055728 + golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c golang.org/x/text v0.3.3 ) diff --git a/go.sum b/go.sum index 0e07309..fdf0228 100644 --- a/go.sum +++ b/go.sum @@ -22,14 +22,14 @@ github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26 h1:8LwryON7jJ8xKzghSTY6KxLMBgZM/Vqc9LlWMqf3pLA= github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw= +github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 h1:YCM8XrQXo9llkohuXw3CBoz2S/T3kVMnb1pM04X23t8= +github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw= github.com/projectdiscovery/fdmax v0.0.2 h1:d0HqNC4kbrMWT669u9W7ksFS7UBvnW0zmgY6FBU45UY= github.com/projectdiscovery/fdmax v0.0.2/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs= github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= -github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693 h1:dBBB/UXLVvy/Onb3YxVyg5ApwY8lRCIXt0owUosvNuI= -github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4= github.com/projectdiscovery/rawhttp v0.0.1 h1:g9lzZPNBZsIAscTS2VLgv8RGQJcFXsOOTlzU49mDeBk= github.com/projectdiscovery/rawhttp v0.0.1/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4= github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= @@ -53,16 +53,14 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA= -golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0= -golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c h1:dk0ukUIHmGHqASjP0iue2261isepFCC6XRCSd1nHgDw= +golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c/go.mod h1:iQL9McJNjoIa5mjH6nYTCTZXUN6RP+XW3eib7Ya3XcI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=