Merge pull request #286 from projectdiscovery/dev

v1.0.7 Release
2026-06-08 16:50:17 +00:00 · 2021-05-26 05:55:53 +05:30
parent c0eb843c75 be85cbc88d
commit bbc8e48ce1
13 changed files with 212 additions and 94 deletions
@@ -12,10 +12,10 @@ jobs:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2.5.2
+        uses: golangci/golangci-lint-action@v2
        with:
          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.31
+          version: latest
          args: --timeout 5m

  build:
@@ -23,8 +23,6 @@ linters-settings:
  #   min-complexity: 15
  goimports:
    local-prefixes: github.com/golangci/golangci-lint
-  golint:
-    min-confidence: 0
  gomnd:
    settings:
      mnd:
@@ -67,7 +65,6 @@ linters:
    - gocritic
    - gofmt
    - goimports
-    - golint
    - gomnd
    - goprintffuncname
    - gosimple
@@ -88,6 +85,7 @@ linters:
    - unused
    - varcheck
    - whitespace
+    - revive

  # don't enable:
  # - depguard
@@ -111,10 +109,3 @@ issues:
  exclude:
    # should have a package comment, unless it's in another file for this package (golint)
    - 'in another file for this package'
-
-# golangci.com configuration
-# https://github.com/golangci/golangci/wiki/Configuration
-service:
-  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
-  prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"
@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http/httputil"
-	"net/url"
 	"strings"

 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/urlutil"
 )

 const (
@@ -74,8 +74,8 @@ func ParseRequest(req string, unsafe bool) (method, path string, headers map[str
 	// Handle case with the full http url in path. In that case,
 	// ignore any host header that we encounter and use the path as request URL
 	if strings.HasPrefix(parts[1], "http") {
-		var parsed *url.URL
-		parsed, err = url.Parse(parts[1])
+		var parsed *urlutil.URL
+		parsed, err = urlutil.Parse(parts[1])
 		if err != nil {
 			err = fmt.Errorf("could not parse request URL: %s", err)
 			return
@@ -8,7 +8,7 @@ import (
 )

 // CSPHeaders is an incomplete list of most common CSP headers
-var CSPHeaders []string = []string{
+var CSPHeaders = []string{
 	"Content-Security-Policy",               // standard
 	"Content-Security-Policy-Report-Only",   // standard
 	"X-Content-Security-Policy-Report-Only", // non - standard
@@ -30,6 +30,8 @@ type ChainItem struct {
 	Request    string `json:"request,omitempty"`
 	Response   string `json:"response,omitempty"`
 	StatusCode int    `json:"status_code,omitempty"`
+	Location   string `json:"location,omitempty"`
+	RequestURL string `json:"request-url,omitempty"`
 }

 // GetHeader value
@@ -79,6 +81,8 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) {
 			Request:    string(chainItem.Request),
 			Response:   string(chainItem.Response),
 			StatusCode: chainItem.StatusCode,
+			Location:   chainItem.Location,
+			RequestURL: chainItem.RequestURL,
 		})
 	}
 	return
@@ -88,3 +92,12 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) {
 func (r *Response) HasChain() bool {
 	return len(r.Chain) > 1
 }
+
+// GetChainLastURL returns the final URL
+func (r *Response) GetChainLastURL() string {
+	if r.HasChain() {
+		lastitem := r.Chain[len(r.Chain)-1]
+		return lastitem.RequestURL
+	}
+	return ""
+}
@@ -11,9 +11,9 @@ import (
 )

 var (
-	cutset                       = "\n\t\v\f\r"
-	reTitle       *regexp.Regexp = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
-	reContentType *regexp.Regexp = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	cutset        = "\n\t\v\f\r"
+	reTitle       = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
+	reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
 )

 // ExtractTitle from a response
@@ -1,32 +1,85 @@
 package httpx

 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
 	"net/http"
 )

 // TLSData contains the relevant Transport Layer Security information
 type TLSData struct {
-	DNSNames         []string `json:"dns_names,omitempty"`
-	Emails           []string `json:"emails,omitempty"`
-	CommonName       []string `json:"common_name,omitempty"`
-	Organization     []string `json:"organization,omitempty"`
-	IssuerCommonName []string `json:"issuer_common_name,omitempty"`
-	IssuerOrg        []string `json:"issuer_organization,omitempty"`
+	TLSVersion               string   `json:"tls_version,omitempty"`
+	ExtensionServerName      string   `json:"extension_server_name,omitempty"`
+	DNSNames                 []string `json:"dns_names,omitempty"`
+	Emails                   []string `json:"emails,omitempty"`
+	CommonName               []string `json:"common_name,omitempty"`
+	Organization             []string `json:"organization,omitempty"`
+	IssuerCommonName         []string `json:"issuer_common_name,omitempty"`
+	IssuerOrg                []string `json:"issuer_organization,omitempty"`
+	FingerprintSHA256        string   `json:"fingerprint_sha256,omitempty"`
+	FingerprintSHA256OpenSSL string   `json:"fingerprint_sha256_openssl,omitempty"`
 }

 // TLSGrab fills the TLSData
 func (h *HTTPX) TLSGrab(r *http.Response) *TLSData {
 	if r.TLS != nil {
 		var tlsdata TLSData
-		for _, certificate := range r.TLS.PeerCertificates {
-			tlsdata.DNSNames = append(tlsdata.DNSNames, certificate.DNSNames...)
-			tlsdata.Emails = append(tlsdata.Emails, certificate.EmailAddresses...)
-			tlsdata.CommonName = append(tlsdata.CommonName, certificate.Subject.CommonName)
-			tlsdata.Organization = append(tlsdata.Organization, certificate.Subject.Organization...)
-			tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, certificate.Issuer.Organization...)
-			tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, certificate.Issuer.CommonName)
+		// Only PeerCertificates[0] contains useful information
+		cert := r.TLS.PeerCertificates[0]
+		tlsdata.DNSNames = append(tlsdata.DNSNames, cert.DNSNames...)
+		tlsdata.Emails = append(tlsdata.Emails, cert.EmailAddresses...)
+		tlsdata.CommonName = append(tlsdata.CommonName, cert.Subject.CommonName)
+		tlsdata.Organization = append(tlsdata.Organization, cert.Subject.Organization...)
+		tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, cert.Issuer.Organization...)
+		tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, cert.Issuer.CommonName)
+		tlsdata.ExtensionServerName = r.TLS.ServerName
+		if v, ok := tlsVersionStringMap[r.TLS.Version]; ok {
+			tlsdata.TLSVersion = v
+		}
+
+		if fingerprintSHA256, err := calculateFingerprints(r); err == nil {
+			tlsdata.FingerprintSHA256 = asHex(fingerprintSHA256)
+			tlsdata.FingerprintSHA256OpenSSL = asOpenSSL(fingerprintSHA256)
 		}
 		return &tlsdata
 	}
 	return nil
 }
+
+var tlsVersionStringMap = map[uint16]string{
+	0x0300: "SSL30",
+	0x0301: "TLS10",
+	0x0302: "TLS11",
+	0x0303: "TLS12",
+	0x0304: "TLS13",
+}
+
+func calculateFingerprints(r *http.Response) (fingerprintSHA256 []byte, err error) {
+	if len(r.TLS.PeerCertificates) == 0 {
+		err = errors.New("no certificates found")
+		return
+	}
+
+	cert := r.TLS.PeerCertificates[0]
+	dataSHA256 := sha256.Sum256(cert.Raw)
+	fingerprintSHA256 = dataSHA256[:]
+	return
+}
+
+func asOpenSSL(b []byte) string {
+	var buf bytes.Buffer
+	for i, f := range b {
+		if i > 0 {
+			fmt.Fprintf(&buf, ":")
+		}
+		fmt.Fprintf(&buf, "%02X", f)
+	}
+	return buf.String()
+}
+
+func asHex(b []byte) string {
+	return hex.EncodeToString(b)
+}
@@ -3,12 +3,15 @@ package stringz
 import (
 	"strconv"
 	"strings"
+
+	"github.com/projectdiscovery/urlutil"
 )

 // TrimProtocol removes the HTTP scheme from an URI
 func TrimProtocol(targetURL string) string {
 	URL := strings.TrimSpace(targetURL)
 	if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") {
+		URL = AddURLDefaultPort(URL)
 		URL = URL[strings.Index(URL, "//")+2:]
 	}

@@ -40,3 +43,31 @@ func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
 	}
 	return
 }
+
+// AddURLDefaultPort add url default port (80/443) from an URI
+// eg:
+// http://foo.com -> http://foo.com:80
+// https://foo.com -> https://foo.com:443
+func AddURLDefaultPort(rawURL string) string {
+	u, err := urlutil.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	return u.String()
+}
+
+// RemoveURLDefaultPort remove url default port (80/443) from an URI
+// eg:
+// http://foo.com:80 -> http://foo.com
+// https://foo.com:443 -> https://foo.com
+func RemoveURLDefaultPort(rawURL string) string {
+	u, err := urlutil.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+
+	if u.Scheme == urlutil.HTTP && u.Port == "80" || u.Scheme == urlutil.HTTPS && u.Port == "443" {
+		u.Port = ""
+	}
+	return u.String()
+}
@@ -7,6 +7,7 @@ require (
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.9
+	github.com/miekg/dns v1.1.42 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/cdncheck v0.0.2
 	github.com/projectdiscovery/clistats v0.0.8
@@ -14,14 +15,17 @@ require (
 	github.com/projectdiscovery/fdmax v0.0.3
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49
+	github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46
 	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/rawhttp v0.0.6
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	github.com/projectdiscovery/wappalyzergo v0.0.3
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
+	github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda // indirect
+	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
+	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
+	golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 // indirect
 	golang.org/x/text v0.3.6
 )
@@ -66,8 +66,9 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
 github.com/microcosm-cc/bluemonday v1.0.9 h1:dpCwruVKoyrULicJwhuY76jB+nIxRVKv/e248Vx/BXg=
 github.com/microcosm-cc/bluemonday v1.0.9/go.mod h1:B2riunDr9benLHghZB7hjIgdwSUzzs0pjCxFrWYEZFU=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
+github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -108,8 +109,8 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49 h1:hzMhw71p7+a5oyd3si9D/PMtqomvdsrd782+9Y1tUVo=
-github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
+github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
+github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
 github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
 github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 h1:veDjJpC3q2PLyuYPS3jNeoYgbHvHPWQhwqRPoCe6YTA=
@@ -124,10 +125,16 @@ github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8
 github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ksVYocwiE0AQ1ytEo=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/projectdiscovery/wappalyzergo v0.0.3 h1:UwaQyl0vbNx+rqwObzvV+YBECQ2NvWAcY/A1KpJyLCA=
-github.com/projectdiscovery/wappalyzergo v0.0.3/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
+github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda h1:q9o7dHZ22CGUWUAzp26nYoCugOkwBDdqMfgMDpIrakM=
+github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
+github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
+github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
+github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
@@ -170,8 +177,10 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -197,8 +206,9 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 h1:lCnv+lfrU9FRPGf8NeRuWAAPjNnema5WtBinMgs1fD8=
+golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -8,11 +8,11 @@ const banner = `
  / __ \/ __/ __/ __ \|   /
 / / / / /_/ /_/ /_/ /   |
 /_/ /_/\__/\__/ .___/_/|_|
-             /_/              v1.0.6
+             /_/              v1.0.7
 `

 // Version is the current version of httpx
-const Version = `v1.0.6`
+const Version = `v1.0.7`

 // showBanner is used to show the banner to the user
 func showBanner() {
@@ -197,7 +197,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
 	flag.StringVar(&options.InputFile, "l", "", "File containing domains")
 	flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
-	flag.BoolVar(&options.OutputMethod, "method", false, "Output method")
+	flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
 	flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
 	flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
 	flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode")
@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
-	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -22,6 +21,8 @@ import (
 	"github.com/pkg/errors"

 	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/urlutil"
+
 	// automatic fd max increase if running as root
 	_ "github.com/projectdiscovery/fdmax/autofdmax"
 	"github.com/projectdiscovery/gologger"
@@ -409,16 +410,23 @@ func (r *Runner) RunEnumeration() {
 	wg := sizedwaitgroup.New(r.options.Threads)

 	r.hm.Scan(func(k, _ []byte) error {
+		t := string(k)
 		var reqs int
+		protocol := r.options.protocol
+		// attempt to parse url as is
+		if u, err := urlutil.Parse(t); err == nil {
+			protocol = u.Scheme
+		}
+
 		if len(r.options.requestURIs) > 0 {
 			for _, p := range r.options.requestURIs {
 				scanopts := r.scanopts.Clone()
 				scanopts.RequestURI = p
-				r.process(string(k), &wg, r.hp, r.options.protocol, scanopts, output)
+				r.process(t, &wg, r.hp, protocol, scanopts, output)
 				reqs++
 			}
 		} else {
-			r.process(string(k), &wg, r.hp, r.options.protocol, &r.scanopts, output)
+			r.process(t, &wg, r.hp, protocol, &r.scanopts, output)
 			reqs++
 		}

@@ -448,7 +456,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 					wg.Add()
 					go func(target, method, protocol string) {
 						defer wg.Done()
-						result := r.analyze(hp, protocol, target, 0, method, scanopts)
+						result := r.analyze(hp, protocol, target, method, scanopts)
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
@@ -470,18 +478,13 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 			}
 		}

-		// the host name shouldn't have any semicolon - in case remove the port
-		semicolonPosition := strings.LastIndex(target, ":")
-		if semicolonPosition > 0 {
-			target = target[:semicolonPosition]
-		}
-
 		for port, wantedProtocol := range customport.Ports {
 			for _, method := range scanopts.Methods {
 				wg.Add()
 				go func(port int, method, protocol string) {
 					defer wg.Done()
-					result := r.analyze(hp, protocol, target, port, method, scanopts)
+					h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
+					result := r.analyze(hp, protocol, h, method, scanopts)
 					output <- result
 					if scanopts.TLSProbe && result.TLSData != nil {
 						scanopts.TLSProbe = false
@@ -527,7 +530,7 @@ func targets(target string) chan string {
 	return results
 }

-func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
+func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result {
 	origProtocol := protocol
 	if protocol == httpx.HTTPorHTTPS {
 		protocol = httpx.HTTPS
@@ -544,49 +547,50 @@ retry:
 		domain = parts[0]
 		customHost = parts[1]
 	}
-
-	URL := fmt.Sprintf("%s://%s", protocol, domain)
-	if port > 0 {
-		URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
-	} else {
-		domainParse := strings.Split(domain, ":")
-		domain = domainParse[0]
-		if len(domainParse) > 1 {
-			port, _ = strconv.Atoi(domainParse[1])
-		}
-	}
+	URL, _ := urlutil.Parse(domain)
+	URL.Scheme = protocol

 	if !scanopts.Unsafe {
-		URL += scanopts.RequestURI
+		URL.RequestURI += scanopts.RequestURI
 	}

-	req, err := hp.NewRequest(method, URL)
+	req, err := hp.NewRequest(method, URL.String())
 	if err != nil {
-		return Result{URL: URL, err: err}
+		return Result{URL: URL.String(), err: err}
 	}
 	if customHost != "" {
 		req.Host = customHost
 	}

+	reqURI := req.URL.RequestURI()
+
 	hp.SetCustomHeaders(req, hp.CustomHeaders)
+	// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
 	if scanopts.RequestBody != "" {
 		req.ContentLength = int64(len(scanopts.RequestBody))
 		req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody))
+	} else {
+		req.ContentLength = 0
+		req.Body = nil
 	}
-
 	var requestDump []byte
 	if scanopts.Unsafe {
-		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), req.RequestURI, req.Header, req.Body, rawhttp.DefaultOptions)
+		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
 		if err != nil {
-			return Result{URL: URL, err: err}
+			return Result{URL: URL.String(), err: err}
 		}
 	} else {
-		// Create a copy on the fly of the request body - ignore errors
+		// Create a copy on the fly of the request body
 		bodyBytes, _ := req.BodyBytes()
 		req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 		requestDump, err = httputil.DumpRequestOut(req.Request, true)
 		if err != nil {
-			return Result{URL: URL, err: err}
+			return Result{URL: URL.String(), err: err}
+		}
+		// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
+		// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
+		if len(bodyBytes) == 0 {
+			req.Body = nil
 		}
 	}

@@ -601,22 +605,18 @@ retry:
 			retried = true
 			goto retry
 		}
-		return Result{URL: URL, err: err}
+		return Result{URL: URL.String(), err: err}
 	}

 	var fullURL string

 	if resp.StatusCode >= 0 {
-		if port > 0 {
-			fullURL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, scanopts.RequestURI)
-		} else {
-			fullURL = fmt.Sprintf("%s://%s%s", protocol, domain, scanopts.RequestURI)
-		}
+		fullURL = req.URL.String()
 	}

 	builder := &strings.Builder{}

-	builder.WriteString(fullURL)
+	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))

 	if scanopts.OutputStatusCode {
 		builder.WriteString(" [")
@@ -725,6 +725,7 @@ retry:

 	pipeline := false
 	if scanopts.Pipeline {
+		port, _ := strconv.Atoi(URL.Port)
 		pipeline = hp.SupportPipeline(protocol, method, domain, port)
 		if pipeline {
 			builder.WriteString(" [pipeline]")
@@ -734,7 +735,7 @@ retry:
 	var http2 bool
 	// if requested probes for http2
 	if scanopts.HTTP2Probe {
-		http2 = hp.SupportHTTP2(protocol, method, URL)
+		http2 = hp.SupportHTTP2(protocol, method, URL.String())
 		if http2 {
 			builder.WriteString(" [http2]")
 		}
@@ -799,12 +800,25 @@ retry:
 		}
 	}

+	var finalURL string
+	if resp.HasChain() {
+		finalURL = resp.GetChainLastURL()
+	}
+
+	if resp.HasChain() {
+		builder.WriteString(" [")
+		if !scanopts.OutputWithNoColor {
+			builder.WriteString(aurora.Magenta(finalURL).String())
+		} else {
+			builder.WriteString(finalURL)
+		}
+		builder.WriteRune(']')
+	}
+
 	// store responses or chain in directory
 	if scanopts.StoreResponse || scanopts.StoreChain {
-		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
-		if port > 0 {
-			domainFile = fmt.Sprintf("%s.%d%s", domain, port, scanopts.RequestURI)
-		}
+		domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
+
 		// On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux
 		// Truncating length at 255
 		if len(domainFile) >= maxFileNameLength {
@@ -833,12 +847,12 @@ retry:
 		}
 	}

-	parsed, err := url.Parse(fullURL)
+	parsed, err := urlutil.Parse(fullURL)
 	if err != nil {
-		return Result{URL: URL, err: errors.Wrap(err, "could not parse url")}
+		return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")}
 	}

-	finalPort := parsed.Port()
+	finalPort := parsed.Port
 	if finalPort == "" {
 		if parsed.Scheme == "http" {
 			finalPort = "80"
@@ -846,7 +860,7 @@ retry:
 			finalPort = "443"
 		}
 	}
-	finalPath := parsed.Path
+	finalPath := parsed.RequestURI
 	if finalPath == "" {
 		finalPath = "/"
 	}
@@ -902,6 +916,7 @@ retry:
 		CDN:              isCDN,
 		ResponseTime:     resp.Duration.String(),
 		Technologies:     technologies,
+		FinalURL:         finalURL,
 	}
 }

@@ -941,6 +956,7 @@ type Result struct {
 	ResponseTime     string            `json:"response-time,omitempty"`
 	Technologies     []string          `json:"technologies,omitempty"`
 	Chain            []httpx.ChainItem `json:"chain,omitempty"`
+	FinalURL         string            `json:"final-url,omitempty"`
 }

 // JSON the result