diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 99bf43b..749babe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,10 +12,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.31 + version: latest args: --timeout 5m build: diff --git a/.golangci.yml b/.golangci.yml index 1a2c5d3..dd00467 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,8 +23,6 @@ linters-settings: # min-complexity: 15 goimports: local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 gomnd: settings: mnd: @@ -67,7 +65,6 @@ linters: - gocritic - gofmt - goimports - - golint - gomnd - goprintffuncname - gosimple @@ -88,6 +85,7 @@ linters: - unused - varcheck - whitespace + - revive # don't enable: # - depguard @@ -111,10 +109,3 @@ issues: exclude: # should have a package comment, unless it's in another file for this package (golint) - 'in another file for this package' - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go index 3a91b9c..8b6367c 100644 --- a/common/httputilz/httputilz.go +++ b/common/httputilz/httputilz.go @@ -5,10 +5,10 @@ import ( "fmt" "io/ioutil" "net/http/httputil" - "net/url" "strings" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/urlutil" ) const ( @@ -74,8 +74,8 @@ func ParseRequest(req string, unsafe bool) (method, path string, headers map[str // Handle case with the full http url in path. In that case, // ignore any host header that we encounter and use the path as request URL if strings.HasPrefix(parts[1], "http") { - var parsed *url.URL - parsed, err = url.Parse(parts[1]) + var parsed *urlutil.URL + parsed, err = urlutil.Parse(parts[1]) if err != nil { err = fmt.Errorf("could not parse request URL: %s", err) return diff --git a/common/httpx/csp.go b/common/httpx/csp.go index 0640d70..7a68ac9 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -8,7 +8,7 @@ import ( ) // CSPHeaders is an incomplete list of most common CSP headers -var CSPHeaders []string = []string{ +var CSPHeaders = []string{ "Content-Security-Policy", // standard "Content-Security-Policy-Report-Only", // standard "X-Content-Security-Policy-Report-Only", // non - standard diff --git a/common/httpx/response.go b/common/httpx/response.go index b14a51a..1d507c4 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -30,6 +30,8 @@ type ChainItem struct { Request string `json:"request,omitempty"` Response string `json:"response,omitempty"` StatusCode int `json:"status_code,omitempty"` + Location string `json:"location,omitempty"` + RequestURL string `json:"request-url,omitempty"` } // GetHeader value @@ -79,6 +81,8 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) { Request: string(chainItem.Request), Response: string(chainItem.Response), StatusCode: chainItem.StatusCode, + Location: chainItem.Location, + RequestURL: chainItem.RequestURL, }) } return @@ -88,3 +92,12 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) { func (r *Response) HasChain() bool { return len(r.Chain) > 1 } + +// GetChainLastURL returns the final URL +func (r *Response) GetChainLastURL() string { + if r.HasChain() { + lastitem := r.Chain[len(r.Chain)-1] + return lastitem.RequestURL + } + return "" +} diff --git a/common/httpx/title.go b/common/httpx/title.go index c80aefa..1edb534 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -11,9 +11,9 @@ import ( ) var ( - cutset = "\n\t\v\f\r" - reTitle *regexp.Regexp = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) - reContentType *regexp.Regexp = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + cutset = "\n\t\v\f\r" + reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) + reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) ) // ExtractTitle from a response diff --git a/common/httpx/tls.go b/common/httpx/tls.go index c74a659..3f1c7c0 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -1,32 +1,85 @@ package httpx import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" "net/http" ) // TLSData contains the relevant Transport Layer Security information type TLSData struct { - DNSNames []string `json:"dns_names,omitempty"` - Emails []string `json:"emails,omitempty"` - CommonName []string `json:"common_name,omitempty"` - Organization []string `json:"organization,omitempty"` - IssuerCommonName []string `json:"issuer_common_name,omitempty"` - IssuerOrg []string `json:"issuer_organization,omitempty"` + TLSVersion string `json:"tls_version,omitempty"` + ExtensionServerName string `json:"extension_server_name,omitempty"` + DNSNames []string `json:"dns_names,omitempty"` + Emails []string `json:"emails,omitempty"` + CommonName []string `json:"common_name,omitempty"` + Organization []string `json:"organization,omitempty"` + IssuerCommonName []string `json:"issuer_common_name,omitempty"` + IssuerOrg []string `json:"issuer_organization,omitempty"` + FingerprintSHA256 string `json:"fingerprint_sha256,omitempty"` + FingerprintSHA256OpenSSL string `json:"fingerprint_sha256_openssl,omitempty"` } // TLSGrab fills the TLSData func (h *HTTPX) TLSGrab(r *http.Response) *TLSData { if r.TLS != nil { var tlsdata TLSData - for _, certificate := range r.TLS.PeerCertificates { - tlsdata.DNSNames = append(tlsdata.DNSNames, certificate.DNSNames...) - tlsdata.Emails = append(tlsdata.Emails, certificate.EmailAddresses...) - tlsdata.CommonName = append(tlsdata.CommonName, certificate.Subject.CommonName) - tlsdata.Organization = append(tlsdata.Organization, certificate.Subject.Organization...) - tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, certificate.Issuer.Organization...) - tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, certificate.Issuer.CommonName) + // Only PeerCertificates[0] contains useful information + cert := r.TLS.PeerCertificates[0] + tlsdata.DNSNames = append(tlsdata.DNSNames, cert.DNSNames...) + tlsdata.Emails = append(tlsdata.Emails, cert.EmailAddresses...) + tlsdata.CommonName = append(tlsdata.CommonName, cert.Subject.CommonName) + tlsdata.Organization = append(tlsdata.Organization, cert.Subject.Organization...) + tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, cert.Issuer.Organization...) + tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, cert.Issuer.CommonName) + tlsdata.ExtensionServerName = r.TLS.ServerName + if v, ok := tlsVersionStringMap[r.TLS.Version]; ok { + tlsdata.TLSVersion = v + } + + if fingerprintSHA256, err := calculateFingerprints(r); err == nil { + tlsdata.FingerprintSHA256 = asHex(fingerprintSHA256) + tlsdata.FingerprintSHA256OpenSSL = asOpenSSL(fingerprintSHA256) } return &tlsdata } return nil } + +var tlsVersionStringMap = map[uint16]string{ + 0x0300: "SSL30", + 0x0301: "TLS10", + 0x0302: "TLS11", + 0x0303: "TLS12", + 0x0304: "TLS13", +} + +func calculateFingerprints(r *http.Response) (fingerprintSHA256 []byte, err error) { + if len(r.TLS.PeerCertificates) == 0 { + err = errors.New("no certificates found") + return + } + + cert := r.TLS.PeerCertificates[0] + dataSHA256 := sha256.Sum256(cert.Raw) + fingerprintSHA256 = dataSHA256[:] + return +} + +func asOpenSSL(b []byte) string { + var buf bytes.Buffer + for i, f := range b { + if i > 0 { + fmt.Fprintf(&buf, ":") + } + fmt.Fprintf(&buf, "%02X", f) + } + return buf.String() +} + +func asHex(b []byte) string { + return hex.EncodeToString(b) +} diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index ae4df9f..f8d0529 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -3,12 +3,15 @@ package stringz import ( "strconv" "strings" + + "github.com/projectdiscovery/urlutil" ) // TrimProtocol removes the HTTP scheme from an URI func TrimProtocol(targetURL string) string { URL := strings.TrimSpace(targetURL) if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") { + URL = AddURLDefaultPort(URL) URL = URL[strings.Index(URL, "//")+2:] } @@ -40,3 +43,31 @@ func SplitByCharAndTrimSpace(s, splitchar string) (result []string) { } return } + +// AddURLDefaultPort add url default port (80/443) from an URI +// eg: +// http://foo.com -> http://foo.com:80 +// https://foo.com -> https://foo.com:443 +func AddURLDefaultPort(rawURL string) string { + u, err := urlutil.Parse(rawURL) + if err != nil { + return rawURL + } + return u.String() +} + +// RemoveURLDefaultPort remove url default port (80/443) from an URI +// eg: +// http://foo.com:80 -> http://foo.com +// https://foo.com:443 -> https://foo.com +func RemoveURLDefaultPort(rawURL string) string { + u, err := urlutil.Parse(rawURL) + if err != nil { + return rawURL + } + + if u.Scheme == urlutil.HTTP && u.Port == "80" || u.Scheme == urlutil.HTTPS && u.Port == "443" { + u.Port = "" + } + return u.String() +} diff --git a/go.mod b/go.mod index b4ca60d..eea382b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.9 + github.com/miekg/dns v1.1.42 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.2 github.com/projectdiscovery/clistats v0.0.8 @@ -14,14 +15,17 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 - github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49 + github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 github.com/projectdiscovery/mapcidr v0.0.7 github.com/projectdiscovery/rawhttp v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.1 - github.com/projectdiscovery/wappalyzergo v0.0.3 + github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 + github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda // indirect + github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 + github.com/projectdiscovery/wappalyzergo v0.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 - golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 + golang.org/x/net v0.0.0-20210525063256-abc453219eb5 + golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 // indirect golang.org/x/text v0.3.6 ) diff --git a/go.sum b/go.sum index 39ea369..623b3a6 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,9 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/microcosm-cc/bluemonday v1.0.9 h1:dpCwruVKoyrULicJwhuY76jB+nIxRVKv/e248Vx/BXg= github.com/microcosm-cc/bluemonday v1.0.9/go.mod h1:B2riunDr9benLHghZB7hjIgdwSUzzs0pjCxFrWYEZFU= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY= +github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -108,8 +109,8 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49 h1:hzMhw71p7+a5oyd3si9D/PMtqomvdsrd782+9Y1tUVo= -github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= +github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0= +github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 h1:veDjJpC3q2PLyuYPS3jNeoYgbHvHPWQhwqRPoCe6YTA= @@ -124,10 +125,16 @@ github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8 github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ksVYocwiE0AQ1ytEo= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/projectdiscovery/wappalyzergo v0.0.3 h1:UwaQyl0vbNx+rqwObzvV+YBECQ2NvWAcY/A1KpJyLCA= -github.com/projectdiscovery/wappalyzergo v0.0.3/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s= +github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= +github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda h1:q9o7dHZ22CGUWUAzp26nYoCugOkwBDdqMfgMDpIrakM= +github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= +github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= +github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= +github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84= +github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= @@ -170,8 +177,10 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -197,8 +206,9 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 h1:lCnv+lfrU9FRPGf8NeRuWAAPjNnema5WtBinMgs1fD8= +golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/runner/banner.go b/runner/banner.go index 972a8f3..295feb6 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.0.6 + /_/ v1.0.7 ` // Version is the current version of httpx -const Version = `v1.0.6` +const Version = `v1.0.7` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/options.go b/runner/options.go index 94d0d50..21d1d37 100644 --- a/runner/options.go +++ b/runner/options.go @@ -197,7 +197,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output") flag.StringVar(&options.InputFile, "l", "", "File containing domains") flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()") - flag.BoolVar(&options.OutputMethod, "method", false, "Output method") + flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") flag.BoolVar(&options.Silent, "silent", false, "Silent mode") flag.BoolVar(&options.Version, "version", false, "Show version of httpx") flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode") diff --git a/runner/runner.go b/runner/runner.go index 876cbbb..2ce477b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "net/http" "net/http/httputil" - "net/url" "os" "path" "regexp" @@ -22,6 +21,8 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/clistats" + "github.com/projectdiscovery/urlutil" + // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/gologger" @@ -409,16 +410,23 @@ func (r *Runner) RunEnumeration() { wg := sizedwaitgroup.New(r.options.Threads) r.hm.Scan(func(k, _ []byte) error { + t := string(k) var reqs int + protocol := r.options.protocol + // attempt to parse url as is + if u, err := urlutil.Parse(t); err == nil { + protocol = u.Scheme + } + if len(r.options.requestURIs) > 0 { for _, p := range r.options.requestURIs { scanopts := r.scanopts.Clone() scanopts.RequestURI = p - r.process(string(k), &wg, r.hp, r.options.protocol, scanopts, output) + r.process(t, &wg, r.hp, protocol, scanopts, output) reqs++ } } else { - r.process(string(k), &wg, r.hp, r.options.protocol, &r.scanopts, output) + r.process(t, &wg, r.hp, protocol, &r.scanopts, output) reqs++ } @@ -448,7 +456,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. wg.Add() go func(target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, 0, method, scanopts) + result := r.analyze(hp, protocol, target, method, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -470,18 +478,13 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } } - // the host name shouldn't have any semicolon - in case remove the port - semicolonPosition := strings.LastIndex(target, ":") - if semicolonPosition > 0 { - target = target[:semicolonPosition] - } - for port, wantedProtocol := range customport.Ports { for _, method := range scanopts.Methods { wg.Add() go func(port int, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, port, method, scanopts) + h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) + result := r.analyze(hp, protocol, h, method, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -527,7 +530,7 @@ func targets(target string) chan string { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS { protocol = httpx.HTTPS @@ -544,49 +547,50 @@ retry: domain = parts[0] customHost = parts[1] } - - URL := fmt.Sprintf("%s://%s", protocol, domain) - if port > 0 { - URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port) - } else { - domainParse := strings.Split(domain, ":") - domain = domainParse[0] - if len(domainParse) > 1 { - port, _ = strconv.Atoi(domainParse[1]) - } - } + URL, _ := urlutil.Parse(domain) + URL.Scheme = protocol if !scanopts.Unsafe { - URL += scanopts.RequestURI + URL.RequestURI += scanopts.RequestURI } - req, err := hp.NewRequest(method, URL) + req, err := hp.NewRequest(method, URL.String()) if err != nil { - return Result{URL: URL, err: err} + return Result{URL: URL.String(), err: err} } if customHost != "" { req.Host = customHost } + reqURI := req.URL.RequestURI() + hp.SetCustomHeaders(req, hp.CustomHeaders) + // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + } else { + req.ContentLength = 0 + req.Body = nil } - var requestDump []byte if scanopts.Unsafe { - requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), req.RequestURI, req.Header, req.Body, rawhttp.DefaultOptions) + requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions) if err != nil { - return Result{URL: URL, err: err} + return Result{URL: URL.String(), err: err} } } else { - // Create a copy on the fly of the request body - ignore errors + // Create a copy on the fly of the request body bodyBytes, _ := req.BodyBytes() req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) requestDump, err = httputil.DumpRequestOut(req.Request, true) if err != nil { - return Result{URL: URL, err: err} + return Result{URL: URL.String(), err: err} + } + // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty + // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) + if len(bodyBytes) == 0 { + req.Body = nil } } @@ -601,22 +605,18 @@ retry: retried = true goto retry } - return Result{URL: URL, err: err} + return Result{URL: URL.String(), err: err} } var fullURL string if resp.StatusCode >= 0 { - if port > 0 { - fullURL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, scanopts.RequestURI) - } else { - fullURL = fmt.Sprintf("%s://%s%s", protocol, domain, scanopts.RequestURI) - } + fullURL = req.URL.String() } builder := &strings.Builder{} - builder.WriteString(fullURL) + builder.WriteString(stringz.RemoveURLDefaultPort(fullURL)) if scanopts.OutputStatusCode { builder.WriteString(" [") @@ -725,6 +725,7 @@ retry: pipeline := false if scanopts.Pipeline { + port, _ := strconv.Atoi(URL.Port) pipeline = hp.SupportPipeline(protocol, method, domain, port) if pipeline { builder.WriteString(" [pipeline]") @@ -734,7 +735,7 @@ retry: var http2 bool // if requested probes for http2 if scanopts.HTTP2Probe { - http2 = hp.SupportHTTP2(protocol, method, URL) + http2 = hp.SupportHTTP2(protocol, method, URL.String()) if http2 { builder.WriteString(" [http2]") } @@ -799,12 +800,25 @@ retry: } } + var finalURL string + if resp.HasChain() { + finalURL = resp.GetChainLastURL() + } + + if resp.HasChain() { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(finalURL).String()) + } else { + builder.WriteString(finalURL) + } + builder.WriteRune(']') + } + // store responses or chain in directory if scanopts.StoreResponse || scanopts.StoreChain { - domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI) - if port > 0 { - domainFile = fmt.Sprintf("%s.%d%s", domain, port, scanopts.RequestURI) - } + domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".") + // On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux // Truncating length at 255 if len(domainFile) >= maxFileNameLength { @@ -833,12 +847,12 @@ retry: } } - parsed, err := url.Parse(fullURL) + parsed, err := urlutil.Parse(fullURL) if err != nil { - return Result{URL: URL, err: errors.Wrap(err, "could not parse url")} + return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")} } - finalPort := parsed.Port() + finalPort := parsed.Port if finalPort == "" { if parsed.Scheme == "http" { finalPort = "80" @@ -846,7 +860,7 @@ retry: finalPort = "443" } } - finalPath := parsed.Path + finalPath := parsed.RequestURI if finalPath == "" { finalPath = "/" } @@ -902,6 +916,7 @@ retry: CDN: isCDN, ResponseTime: resp.Duration.String(), Technologies: technologies, + FinalURL: finalURL, } } @@ -941,6 +956,7 @@ type Result struct { ResponseTime string `json:"response-time,omitempty"` Technologies []string `json:"technologies,omitempty"` Chain []httpx.ChainItem `json:"chain,omitempty"` + FinalURL string `json:"final-url,omitempty"` } // JSON the result