admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #130 from projectdiscovery/remove-automatic-cdn

Browse Source 
making cdn check optional
This commit is contained in:
bauthard
2020-10-04 13:08:09 +05:30
committed by GitHub
parent cfc9738f56 785b2b2b65
commit 49347f50fe
6 changed files with 28 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/httpx/httpx.go
+4 -3
View File
@@ -47,6 +47,7 @@ func main() {
httpxOptions.HTTPProxy = options.HTTPProxy
httpxOptions.Unsafe = options.Unsafe
httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
httpxOptions.CdnCheck = options.OutputCDN
var key, value string
httpxOptions.CustomHeaders = make(map[string]string)
@@ -556,8 +557,8 @@ retry:
builder.WriteString(fmt.Sprintf(" [%s]", cnames[0]))
}
isCDN := hp.CdnCheck(ip)
if scanopts.OutputCDN && isCDN {
isCDN, err := hp.CdnCheck(ip)
if scanopts.OutputCDN && isCDN && err == nil {
builder.WriteString(" [cdn]")
}
@@ -635,7 +636,7 @@ type Result struct {
WebSocket bool `json:"websocket,omitempty"`
Pipeline bool `json:"pipeline,omitempty"`
HTTP2 bool `json:"http2"`
CDN bool `json:"cdn"`
CDN bool `json:"cdn,omitempty"`
Duration time.Duration `json:"duration"`
}
common/httpx/cdn.go
+9 -4
View File
@@ -1,10 +1,15 @@
package httpx
import "net"
import (
"fmt"
"net"
)
// CdnCheck verifies if the given ip is part of Cdn ranges
func (h *HTTPX) CdnCheck(ip string) bool {
ok, err := h.cdn.Check(net.ParseIP((ip)))
func (h *HTTPX) CdnCheck(ip string) (bool, error) {
if h.cdn == nil {
return false, fmt.Errorf("cdn client not configured")
}
return ok && err == nil
return h.cdn.Check(net.ParseIP((ip)))
}
common/httpx/httpx.go
+5 -3
View File
@@ -103,9 +103,11 @@ func New(options *Options) (*HTTPX, error) {
httpx.htmlPolicy = bluemonday.NewPolicy()
httpx.CustomHeaders = httpx.Options.CustomHeaders
httpx.RequestOverride = &options.RequestOverride
httpx.cdn, err = cdncheck.New()
if err != nil {
return nil, fmt.Errorf("could not create cdn check: %s", err)
if options.CdnCheck {
httpx.cdn, err = cdncheck.NewWithCache()
if err != nil {
return nil, fmt.Errorf("could not create cdn check: %s", err)
}
}
return httpx, nil
common/httpx/option.go
+2 -1
View File
@@ -11,6 +11,7 @@ type Options struct {
HTTPProxy string
SocksProxy string
Threads int
CdnCheck bool
// Timeout is the maximum time to wait for the request
Timeout time.Duration
// RetryMax is the maximum number of retries
@@ -35,6 +36,7 @@ var DefaultOptions = Options{
Timeout: 30 * time.Second,
RetryMax: 5,
Unsafe: false,
CdnCheck: true,
// VHOSTs options
VHostIgnoreStatusCode: false,
VHostIgnoreContentLength: true,
@@ -43,5 +45,4 @@ var DefaultOptions = Options{
VHostStripHTML: false,
VHostSimilarityRatio: 85,
DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)",
// Smuggling Options
}
go.mod
+2 -2
View File
@@ -8,7 +8,7 @@ require (
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/microcosm-cc/bluemonday v1.0.4
github.com/miekg/dns v1.1.31
github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935
github.com/projectdiscovery/fdmax v0.0.2
github.com/projectdiscovery/gologger v1.0.1
github.com/projectdiscovery/mapcidr v0.0.4
@@ -16,6 +16,6 @@ require (
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.2.1
golang.org/x/net v0.0.0-20200927032502-5d4f70055728
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c
golang.org/x/text v0.3.3
)
go.sum
+6 -8
View File
@@ -22,14 +22,14 @@ github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26 h1:8LwryON7jJ8xKzghSTY6KxLMBgZM/Vqc9LlWMqf3pLA=
github.com/projectdiscovery/cdncheck v0.0.0-20200910082712-19e1db650e26/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw=
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935 h1:YCM8XrQXo9llkohuXw3CBoz2S/T3kVMnb1pM04X23t8=
github.com/projectdiscovery/cdncheck v0.0.0-20201003183750-5bc57c383935/go.mod h1:+CNeKlAVwecauIkA+PBNoA7zXGm4MZhL3KKFkkpIaZw=
github.com/projectdiscovery/fdmax v0.0.2 h1:d0HqNC4kbrMWT669u9W7ksFS7UBvnW0zmgY6FBU45UY=
github.com/projectdiscovery/fdmax v0.0.2/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs=
github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693 h1:dBBB/UXLVvy/Onb3YxVyg5ApwY8lRCIXt0owUosvNuI=
github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/rawhttp v0.0.1 h1:g9lzZPNBZsIAscTS2VLgv8RGQJcFXsOOTlzU49mDeBk=
github.com/projectdiscovery/rawhttp v0.0.1/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
@@ -53,16 +53,14 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA=
golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0=
golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c h1:dk0ukUIHmGHqASjP0iue2261isepFCC6XRCSd1nHgDw=
golang.org/x/net v0.0.0-20201002202402-0a1ea396d57c/go.mod h1:iQL9McJNjoIa5mjH6nYTCTZXUN6RP+XW3eib7Ya3XcI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=