From d7fb3384fed6f7b626e8493b086702db0d65feab Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Fri, 9 Apr 2021 16:12:01 +0200
Subject: [PATCH 01/23] improving certificate parsing

https://github.com/projectdiscovery/httpx/issues/221
---
 common/httpx/tls.go | 105 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 92 insertions(+), 13 deletions(-)

diff --git a/common/httpx/tls.go b/common/httpx/tls.go
index c74a659..8ddea7d 100644
--- a/common/httpx/tls.go
+++ b/common/httpx/tls.go
@@ -1,32 +1,111 @@
 package httpx
 
 import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"errors"
+	"fmt"
 	"net/http"
 )
 
 // TLSData contains the relevant Transport Layer Security information
 type TLSData struct {
-	DNSNames         []string `json:"dns_names,omitempty"`
-	Emails           []string `json:"emails,omitempty"`
-	CommonName       []string `json:"common_name,omitempty"`
-	Organization     []string `json:"organization,omitempty"`
-	IssuerCommonName []string `json:"issuer_common_name,omitempty"`
-	IssuerOrg        []string `json:"issuer_organization,omitempty"`
+	TLSVersion               string              `json:"tls_version,omitempty"`
+	CipherSuite              string              `json:"cipher_suite,omitempty"`
+	NegotiatedProtocol       string              `json:"negotiated_protocol,omitempty"`
+	ExtensionServerName      string              `json:"extension_server_name,omitempty"`
+	OCSPResponse             []byte              `json:"ocsp_response,omitempty"`
+	DNSNames                 []string            `json:"dns_names,omitempty"`
+	Emails                   []string            `json:"emails,omitempty"`
+	CommonName               []string            `json:"common_name,omitempty"`
+	Organization             []string            `json:"organization,omitempty"`
+	IssuerCommonName         []string            `json:"issuer_common_name,omitempty"`
+	IssuerOrg                []string            `json:"issuer_organization,omitempty"`
+	FingerprintMD5           string              `json:"fingerprint_md5,omitempty"`
+	FingerprintSHA1          string              `json:"fingerprint_sha1,omitempty"`
+	FingerprintSHA256        string              `json:"fingerprint_sha256,omitempty"`
+	FingerprintMD5OpenSSL    string              `json:"fingerprint_md5_openssl,omitempty"`
+	FingerprintSHA1OpenSSL   string              `json:"fingerprint_sha1_openssl,omitempty"`
+	FingerprintSHA256OpenSSL string              `json:"fingerprint_sha256_openssl,omitempty"`
+	RawCertificateChain      []*x509.Certificate `json:"raw_certificate_chain,omitempty"`
 }
 
 // TLSGrab fills the TLSData
 func (h *HTTPX) TLSGrab(r *http.Response) *TLSData {
 	if r.TLS != nil {
 		var tlsdata TLSData
-		for _, certificate := range r.TLS.PeerCertificates {
-			tlsdata.DNSNames = append(tlsdata.DNSNames, certificate.DNSNames...)
-			tlsdata.Emails = append(tlsdata.Emails, certificate.EmailAddresses...)
-			tlsdata.CommonName = append(tlsdata.CommonName, certificate.Subject.CommonName)
-			tlsdata.Organization = append(tlsdata.Organization, certificate.Subject.Organization...)
-			tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, certificate.Issuer.Organization...)
-			tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, certificate.Issuer.CommonName)
+		// Only PeerCertificates[0] contains useful information
+		cert := r.TLS.PeerCertificates[0]
+		tlsdata.DNSNames = append(tlsdata.DNSNames, cert.DNSNames...)
+		tlsdata.Emails = append(tlsdata.Emails, cert.EmailAddresses...)
+		tlsdata.CommonName = append(tlsdata.CommonName, cert.Subject.CommonName)
+		tlsdata.Organization = append(tlsdata.Organization, cert.Subject.Organization...)
+		tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, cert.Issuer.Organization...)
+		tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, cert.Issuer.CommonName)
+		tlsdata.CipherSuite = tls.CipherSuiteName(r.TLS.CipherSuite)
+		tlsdata.NegotiatedProtocol = r.TLS.NegotiatedProtocol
+		tlsdata.ExtensionServerName = r.TLS.ServerName
+		tlsdata.OCSPResponse = r.TLS.OCSPResponse
+		if v, ok := tlsVersionStringMap[r.TLS.Version]; ok {
+			tlsdata.TLSVersion = v
 		}
+
+		if fingerprintMD5, fingerprintSHA1, fingerprintSHA256, err := calculatFingerprints(r); err == nil {
+			tlsdata.FingerprintMD5 = asHex(fingerprintMD5)
+			tlsdata.FingerprintSHA1 = asHex(fingerprintSHA1)
+			tlsdata.FingerprintSHA256 = asHex(fingerprintSHA256)
+			tlsdata.FingerprintMD5OpenSSL = asOpenSSL(fingerprintMD5)
+			tlsdata.FingerprintSHA1OpenSSL = asOpenSSL(fingerprintSHA1)
+			tlsdata.FingerprintSHA256OpenSSL = asOpenSSL(fingerprintSHA256)
+		}
+
+		tlsdata.RawCertificateChain = r.TLS.PeerCertificates
+
 		return &tlsdata
 	}
 	return nil
 }
+
+var tlsVersionStringMap = map[uint16]string{
+	0x0300: "SSL30",
+	0x0301: "TLS10",
+	0x0302: "TLS11",
+	0x0303: "TLS12",
+	0x0304: "TLS13",
+}
+
+func calculatFingerprints(r *http.Response) (fingerprintMD5, fingerprintSHA1, fingerprintSHA256 []byte, err error) {
+	if len(r.TLS.PeerCertificates) == 0 {
+		err = errors.New("no certificates found")
+		return
+	}
+
+	cert := r.TLS.PeerCertificates[0]
+	dataMD5 := md5.Sum(cert.Raw)
+	fingerprintMD5 = dataMD5[:]
+	dataSHA1 := sha1.Sum(cert.Raw)
+	fingerprintSHA1 = dataSHA1[:]
+	dataSHA256 := sha256.Sum256(cert.Raw)
+	fingerprintSHA256 = dataSHA256[:]
+	return
+}
+
+func asOpenSSL(b []byte) string {
+	var buf bytes.Buffer
+	for i, f := range b {
+		if i > 0 {
+			fmt.Fprintf(&buf, ":")
+		}
+		fmt.Fprintf(&buf, "%02X", f)
+	}
+	return buf.String()
+}
+
+func asHex(b []byte) string {
+	return hex.EncodeToString(b)
+}

From d18cfcc1ae8009157b4b519923ec9fa816d5217f Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Sat, 8 May 2021 20:38:06 +0200
Subject: [PATCH 02/23] adding final url support

---
 common/httpx/response.go | 13 +++++++++++++
 go.mod                   |  2 +-
 go.sum                   |  2 ++
 runner/runner.go         | 12 ++++++++++++
 4 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/common/httpx/response.go b/common/httpx/response.go
index b14a51a..ab7bd0c 100644
--- a/common/httpx/response.go
+++ b/common/httpx/response.go
@@ -30,6 +30,8 @@ type ChainItem struct {
 	Request    string `json:"request,omitempty"`
 	Response   string `json:"response,omitempty"`
 	StatusCode int    `json:"status_code,omitempty"`
+	Location   string `json:"location,omitempty"`
+	RequestURL string `json:"request-url,omitempty"`
 }
 
 // GetHeader value
@@ -79,6 +81,8 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) {
 			Request:    string(chainItem.Request),
 			Response:   string(chainItem.Response),
 			StatusCode: chainItem.StatusCode,
+			Location:   chainItem.Location,
+			RequestURL: chainItem.RequestURL,
 		})
 	}
 	return
@@ -88,3 +92,12 @@ func (r *Response) GetChainAsSlice() (chain []ChainItem) {
 func (r *Response) HasChain() bool {
 	return len(r.Chain) > 1
 }
+
+// HasChain redirects
+func (r *Response) GetChainLastURL() string {
+	if r.HasChain() {
+		lastitem := r.Chain[len(r.Chain)-1]
+		return lastitem.RequestURL
+	}
+	return ""
+}
diff --git a/go.mod b/go.mod
index 1766439..310688d 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/projectdiscovery/fdmax v0.0.3
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
-	github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49
+	github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46
 	github.com/projectdiscovery/mapcidr v0.0.6
 	github.com/projectdiscovery/rawhttp v0.0.6
diff --git a/go.sum b/go.sum
index dd313b4..78855d2 100644
--- a/go.sum
+++ b/go.sum
@@ -108,6 +108,8 @@ github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
 github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49 h1:hzMhw71p7+a5oyd3si9D/PMtqomvdsrd782+9Y1tUVo=
 github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
+github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
+github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
 github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
 github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 h1:veDjJpC3q2PLyuYPS3jNeoYgbHvHPWQhwqRPoCe6YTA=
diff --git a/runner/runner.go b/runner/runner.go
index 876cbbb..9edb989 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -799,6 +799,16 @@ retry:
 		}
 	}
 
+	var finalURL string
+	if resp.HasChain() {
+		finalURL = resp.GetChainLastURL()
+	}
+
+	if resp.HasChain() {
+		builder.WriteString(" [" + finalURL + "]")
+
+	}
+
 	// store responses or chain in directory
 	if scanopts.StoreResponse || scanopts.StoreChain {
 		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
@@ -902,6 +912,7 @@ retry:
 		CDN:              isCDN,
 		ResponseTime:     resp.Duration.String(),
 		Technologies:     technologies,
+		FinalURL:         finalURL,
 	}
 }
 
@@ -941,6 +952,7 @@ type Result struct {
 	ResponseTime     string            `json:"response-time,omitempty"`
 	Technologies     []string          `json:"technologies,omitempty"`
 	Chain            []httpx.ChainItem `json:"chain,omitempty"`
+	FinalURL         string            `json:"final-url,omitempty"`
 }
 
 // JSON the result

From 7f0c2d70028be968585cf31e0df6437534a70af1 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Sat, 8 May 2021 20:40:55 +0200
Subject: [PATCH 03/23] adding comment

---
 common/httpx/response.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/httpx/response.go b/common/httpx/response.go
index ab7bd0c..1d507c4 100644
--- a/common/httpx/response.go
+++ b/common/httpx/response.go
@@ -93,7 +93,7 @@ func (r *Response) HasChain() bool {
 	return len(r.Chain) > 1
 }
 
-// HasChain redirects
+// GetChainLastURL returns the final URL
 func (r *Response) GetChainLastURL() string {
 	if r.HasChain() {
 		lastitem := r.Chain[len(r.Chain)-1]

From 268cf3d03beedfcb9b1cb9c74dfc841e5f37905f Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Sat, 8 May 2021 20:43:34 +0200
Subject: [PATCH 04/23] misc lint

---
 runner/runner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runner/runner.go b/runner/runner.go
index 9edb989..d8ac868 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -806,7 +806,6 @@ retry:
 
 	if resp.HasChain() {
 		builder.WriteString(" [" + finalURL + "]")
-
 	}
 
 	// store responses or chain in directory

From d258ac3a40ccfe9132878257366ac39b1e34a00b Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sun, 9 May 2021 16:21:31 +0530
Subject: [PATCH 05/23] version update

---
 runner/banner.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/runner/banner.go b/runner/banner.go
index 972a8f3..295feb6 100644
--- a/runner/banner.go
+++ b/runner/banner.go
@@ -8,11 +8,11 @@ const banner = `
   / __ \/ __/ __/ __ \|   /
  / / / / /_/ /_/ /_/ /   |
 /_/ /_/\__/\__/ .___/_/|_|
-             /_/              v1.0.6
+             /_/              v1.0.7
 `
 
 // Version is the current version of httpx
-const Version = `v1.0.6`
+const Version = `v1.0.7`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

From 7d88fa33f7dbd9d188aeb92964f9b5b6317a78cc Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Thu, 13 May 2021 19:23:11 +0530
Subject: [PATCH 06/23] Help menu update

---
 runner/options.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runner/options.go b/runner/options.go
index 94d0d50..21d1d37 100644
--- a/runner/options.go
+++ b/runner/options.go
@@ -197,7 +197,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.JSONOutput, "json", false, "JSON Output")
 	flag.StringVar(&options.InputFile, "l", "", "File containing domains")
 	flag.StringVar(&options.Methods, "x", "", "Request Methods, use ALL to check all verbs ()")
-	flag.BoolVar(&options.OutputMethod, "method", false, "Output method")
+	flag.BoolVar(&options.OutputMethod, "method", false, "Display request method")
 	flag.BoolVar(&options.Silent, "silent", false, "Silent mode")
 	flag.BoolVar(&options.Version, "version", false, "Show version of httpx")
 	flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode")

From c1ee06a6d8104e7becead62b17deb7f3bcc56c28 Mon Sep 17 00:00:00 2001
From: becivells <732903873@qq.com>
Date: Fri, 14 May 2021 15:04:49 +0800
Subject: [PATCH 07/23] Add default port for URL to solve automatic use of
 HTTPS in function string.TrimProtocol

---
 common/stringz/stringz.go | 53 +++++++++++++++++++++++++++++++++++++++
 runner/runner.go          |  2 +-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
index ae4df9f..2f6f0dd 100644
--- a/common/stringz/stringz.go
+++ b/common/stringz/stringz.go
@@ -1,14 +1,19 @@
 package stringz
 
 import (
+	"fmt"
+	"net/url"
 	"strconv"
 	"strings"
+
+	"github.com/projectdiscovery/httpx/common/httpx"
 )
 
 // TrimProtocol removes the HTTP scheme from an URI
 func TrimProtocol(targetURL string) string {
 	URL := strings.TrimSpace(targetURL)
 	if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") {
+		URL = AddURLDefaultPort(URL)
 		URL = URL[strings.Index(URL, "//")+2:]
 	}
 
@@ -40,3 +45,51 @@ func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
 	}
 	return
 }
+
+// AddURLDefaultPort add url default port (80/443) from an URI
+// eg:
+// http://foo.com -> http://foo.com:80
+// https://foo.com -> https://foo.com:443
+func AddURLDefaultPort(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	// http://[::]
+	if strings.HasPrefix(u.Host, "[") && strings.HasSuffix(u.Host, "]") {
+		if u.Scheme == httpx.HTTPS {
+			u.Host = fmt.Sprintf("%s:%s", u.Host, "443")
+		} else {
+			u.Host = fmt.Sprintf("%s:%s", u.Host, "80")
+		}
+	}
+	// http://foo.com:81
+	// http://foo.com
+	// http://[::]:80
+	if strings.LastIndexByte(u.Host, ':') == -1 {
+		if u.Scheme == httpx.HTTPS {
+			u.Host = fmt.Sprintf("%s:%s", u.Host, "443")
+		} else {
+			u.Host = fmt.Sprintf("%s:%s", u.Host, "80")
+		}
+	}
+	return u.String()
+}
+
+// RemoveURLDefaultPort remove url default port (80/443) from an URI
+// eg:
+// http://foo.com:80 -> http://foo.com
+// https://foo.com:443 -> https://foo.com
+func RemoveURLDefaultPort(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	colon := strings.LastIndexByte(u.Host, ':')
+	if colon != -1 {
+		if (u.Scheme == "https" && u.Host[colon+1:] == "443") || u.Scheme == "http" && u.Host[colon+1:] == "80" {
+			u.Host = u.Host[:colon]
+		}
+	}
+	return u.String()
+}
diff --git a/runner/runner.go b/runner/runner.go
index 876cbbb..a4f4d7b 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -616,7 +616,7 @@ retry:
 
 	builder := &strings.Builder{}
 
-	builder.WriteString(fullURL)
+	builder.WriteString(stringz.RemoveURLDefaultPort(fullURL))
 
 	if scanopts.OutputStatusCode {
 		builder.WriteString(" [")

From 94758080372563a173deb1c29646e8ae3d659f0d Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 15 May 2021 22:34:28 +0530
Subject: [PATCH 08/23] Adding color to finalURL

---
 runner/runner.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/runner/runner.go b/runner/runner.go
index d8ac868..eecf14a 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -804,8 +804,15 @@ retry:
 		finalURL = resp.GetChainLastURL()
 	}
 
+
 	if resp.HasChain() {
-		builder.WriteString(" [" + finalURL + "]")
+		builder.WriteString(" [")
+		if !scanopts.OutputWithNoColor {
+			builder.WriteString(aurora.Magenta(finalURL).String())
+		} else {
+			builder.WriteString(finalURL)
+		}
+		builder.WriteRune(']')
 	}
 
 	// store responses or chain in directory

From 13d940ce77e225c8932cef2f58418268c5b830da Mon Sep 17 00:00:00 2001
From: sandeep <8293321+ehsandeep@users.noreply.github.com>
Date: Sat, 15 May 2021 22:41:21 +0530
Subject: [PATCH 09/23] misc changes

---
 runner/runner.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/runner/runner.go b/runner/runner.go
index eecf14a..541467c 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -804,7 +804,6 @@ retry:
 		finalURL = resp.GetChainLastURL()
 	}
 
-
 	if resp.HasChain() {
 		builder.WriteString(" [")
 		if !scanopts.OutputWithNoColor {

From 934e10401521272223c043b7232faa634fedc1fc Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Sun, 23 May 2021 02:02:02 +0530
Subject: [PATCH 10/23] Removed some unnecessary data from json

---
 common/httpx/tls.go | 50 +++++++++++----------------------------------
 1 file changed, 12 insertions(+), 38 deletions(-)

diff --git a/common/httpx/tls.go b/common/httpx/tls.go
index 8ddea7d..3f1c7c0 100644
--- a/common/httpx/tls.go
+++ b/common/httpx/tls.go
@@ -2,11 +2,7 @@ package httpx
 
 import (
 	"bytes"
-	"crypto/md5"
-	"crypto/sha1"
 	"crypto/sha256"
-	"crypto/tls"
-	"crypto/x509"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -15,24 +11,16 @@ import (
 
 // TLSData contains the relevant Transport Layer Security information
 type TLSData struct {
-	TLSVersion               string              `json:"tls_version,omitempty"`
-	CipherSuite              string              `json:"cipher_suite,omitempty"`
-	NegotiatedProtocol       string              `json:"negotiated_protocol,omitempty"`
-	ExtensionServerName      string              `json:"extension_server_name,omitempty"`
-	OCSPResponse             []byte              `json:"ocsp_response,omitempty"`
-	DNSNames                 []string            `json:"dns_names,omitempty"`
-	Emails                   []string            `json:"emails,omitempty"`
-	CommonName               []string            `json:"common_name,omitempty"`
-	Organization             []string            `json:"organization,omitempty"`
-	IssuerCommonName         []string            `json:"issuer_common_name,omitempty"`
-	IssuerOrg                []string            `json:"issuer_organization,omitempty"`
-	FingerprintMD5           string              `json:"fingerprint_md5,omitempty"`
-	FingerprintSHA1          string              `json:"fingerprint_sha1,omitempty"`
-	FingerprintSHA256        string              `json:"fingerprint_sha256,omitempty"`
-	FingerprintMD5OpenSSL    string              `json:"fingerprint_md5_openssl,omitempty"`
-	FingerprintSHA1OpenSSL   string              `json:"fingerprint_sha1_openssl,omitempty"`
-	FingerprintSHA256OpenSSL string              `json:"fingerprint_sha256_openssl,omitempty"`
-	RawCertificateChain      []*x509.Certificate `json:"raw_certificate_chain,omitempty"`
+	TLSVersion               string   `json:"tls_version,omitempty"`
+	ExtensionServerName      string   `json:"extension_server_name,omitempty"`
+	DNSNames                 []string `json:"dns_names,omitempty"`
+	Emails                   []string `json:"emails,omitempty"`
+	CommonName               []string `json:"common_name,omitempty"`
+	Organization             []string `json:"organization,omitempty"`
+	IssuerCommonName         []string `json:"issuer_common_name,omitempty"`
+	IssuerOrg                []string `json:"issuer_organization,omitempty"`
+	FingerprintSHA256        string   `json:"fingerprint_sha256,omitempty"`
+	FingerprintSHA256OpenSSL string   `json:"fingerprint_sha256_openssl,omitempty"`
 }
 
 // TLSGrab fills the TLSData
@@ -47,25 +35,15 @@ func (h *HTTPX) TLSGrab(r *http.Response) *TLSData {
 		tlsdata.Organization = append(tlsdata.Organization, cert.Subject.Organization...)
 		tlsdata.IssuerOrg = append(tlsdata.IssuerOrg, cert.Issuer.Organization...)
 		tlsdata.IssuerCommonName = append(tlsdata.IssuerCommonName, cert.Issuer.CommonName)
-		tlsdata.CipherSuite = tls.CipherSuiteName(r.TLS.CipherSuite)
-		tlsdata.NegotiatedProtocol = r.TLS.NegotiatedProtocol
 		tlsdata.ExtensionServerName = r.TLS.ServerName
-		tlsdata.OCSPResponse = r.TLS.OCSPResponse
 		if v, ok := tlsVersionStringMap[r.TLS.Version]; ok {
 			tlsdata.TLSVersion = v
 		}
 
-		if fingerprintMD5, fingerprintSHA1, fingerprintSHA256, err := calculatFingerprints(r); err == nil {
-			tlsdata.FingerprintMD5 = asHex(fingerprintMD5)
-			tlsdata.FingerprintSHA1 = asHex(fingerprintSHA1)
+		if fingerprintSHA256, err := calculateFingerprints(r); err == nil {
 			tlsdata.FingerprintSHA256 = asHex(fingerprintSHA256)
-			tlsdata.FingerprintMD5OpenSSL = asOpenSSL(fingerprintMD5)
-			tlsdata.FingerprintSHA1OpenSSL = asOpenSSL(fingerprintSHA1)
 			tlsdata.FingerprintSHA256OpenSSL = asOpenSSL(fingerprintSHA256)
 		}
-
-		tlsdata.RawCertificateChain = r.TLS.PeerCertificates
-
 		return &tlsdata
 	}
 	return nil
@@ -79,17 +57,13 @@ var tlsVersionStringMap = map[uint16]string{
 	0x0304: "TLS13",
 }
 
-func calculatFingerprints(r *http.Response) (fingerprintMD5, fingerprintSHA1, fingerprintSHA256 []byte, err error) {
+func calculateFingerprints(r *http.Response) (fingerprintSHA256 []byte, err error) {
 	if len(r.TLS.PeerCertificates) == 0 {
 		err = errors.New("no certificates found")
 		return
 	}
 
 	cert := r.TLS.PeerCertificates[0]
-	dataMD5 := md5.Sum(cert.Raw)
-	fingerprintMD5 = dataMD5[:]
-	dataSHA1 := sha1.Sum(cert.Raw)
-	fingerprintSHA1 = dataSHA1[:]
 	dataSHA256 := sha256.Sum256(cert.Raw)
 	fingerprintSHA256 = dataSHA256[:]
 	return

From b1e1fff01b47b2d8f6fb40af6ffe2ad3750ab1d5 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Sun, 23 May 2021 18:15:28 +0200
Subject: [PATCH 11/23] Better url parsing - 277

---
 runner/runner.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/runner/runner.go b/runner/runner.go
index baad5ad..d613219 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -410,7 +410,13 @@ func (r *Runner) RunEnumeration() {
 
 	r.hm.Scan(func(k, _ []byte) error {
 		var reqs int
-		if len(r.options.requestURIs) > 0 {
+		// in case of full URLs use them as is
+		if u, err := url.Parse(string(k)); err == nil {
+			scanopts := r.scanopts.Clone()
+			scanopts.RequestURI = u.RequestURI()
+			r.process(string(k), &wg, r.hp, u.Scheme, scanopts, output)
+			reqs++
+		} else if len(r.options.requestURIs) > 0 {
 			for _, p := range r.options.requestURIs {
 				scanopts := r.scanopts.Clone()
 				scanopts.RequestURI = p
@@ -552,6 +558,11 @@ retry:
 		domainParse := strings.Split(domain, ":")
 		domain = domainParse[0]
 		if len(domainParse) > 1 {
+			// consider the port till the next /
+			if strings.Contains(domainParse[1], "/") {
+				iForwardSlash := strings.Index(domainParse[1], "/")
+				domainParse[1] = domainParse[1][:iForwardSlash]
+			}
 			port, _ = strconv.Atoi(domainParse[1])
 		}
 	}

From 8d8075f138128a8c10a1dda912cb401e3199d694 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Sun, 23 May 2021 22:40:29 +0200
Subject: [PATCH 12/23] additional checks on URL parsing

---
 runner/runner.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/runner/runner.go b/runner/runner.go
index d613219..b19f5db 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -409,22 +409,23 @@ func (r *Runner) RunEnumeration() {
 	wg := sizedwaitgroup.New(r.options.Threads)
 
 	r.hm.Scan(func(k, _ []byte) error {
+		t := string(k)
 		var reqs int
-		// in case of full URLs use them as is
-		if u, err := url.Parse(string(k)); err == nil {
+		// full url should have either scheme (eg http-https) or query path (eg /)
+		if u, err := url.Parse(t); err == nil && (u.Scheme != "" || u.RequestURI() != t) {
 			scanopts := r.scanopts.Clone()
 			scanopts.RequestURI = u.RequestURI()
-			r.process(string(k), &wg, r.hp, u.Scheme, scanopts, output)
+			r.process(t, &wg, r.hp, u.Scheme, scanopts, output)
 			reqs++
 		} else if len(r.options.requestURIs) > 0 {
 			for _, p := range r.options.requestURIs {
 				scanopts := r.scanopts.Clone()
 				scanopts.RequestURI = p
-				r.process(string(k), &wg, r.hp, r.options.protocol, scanopts, output)
+				r.process(t, &wg, r.hp, r.options.protocol, scanopts, output)
 				reqs++
 			}
 		} else {
-			r.process(string(k), &wg, r.hp, r.options.protocol, &r.scanopts, output)
+			r.process(t, &wg, r.hp, r.options.protocol, &r.scanopts, output)
 			reqs++
 		}
 

From b114674b2dac548028783cd0a6e5ab33689f8e0b Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Mon, 24 May 2021 01:31:25 +0200
Subject: [PATCH 13/23] various urls parsing changes

---
 runner/runner.go | 67 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/runner/runner.go b/runner/runner.go
index b19f5db..c55d3b7 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -411,21 +411,40 @@ func (r *Runner) RunEnumeration() {
 	r.hm.Scan(func(k, _ []byte) error {
 		t := string(k)
 		var reqs int
-		// full url should have either scheme (eg http-https) or query path (eg /)
-		if u, err := url.Parse(t); err == nil && (u.Scheme != "" || u.RequestURI() != t) {
-			scanopts := r.scanopts.Clone()
-			scanopts.RequestURI = u.RequestURI()
-			r.process(t, &wg, r.hp, u.Scheme, scanopts, output)
-			reqs++
-		} else if len(r.options.requestURIs) > 0 {
+		protocol := r.options.protocol
+		requestURI := ""
+		// attempt to parse url as is
+		if u, err := url.Parse(t); err == nil {
+			switch u.Scheme {
+			case httpx.HTTP:
+				protocol = httpx.HTTP
+			case httpx.HTTPS:
+				protocol = httpx.HTTPS
+			}
+			requestURI = u.RequestURI()
+			if !strings.HasPrefix(requestURI, "/") {
+				// if requestURI doesn't start with "/" attempts to reparse with generic http protocol
+				if u, err := url.Parse("http://" + t); err == nil {
+					requestURI = u.RequestURI()
+				}
+			}
+			// if it's only "/" skip it
+			if requestURI == "/" {
+				requestURI = ""
+			}
+		}
+
+		if len(r.options.requestURIs) > 0 {
 			for _, p := range r.options.requestURIs {
 				scanopts := r.scanopts.Clone()
-				scanopts.RequestURI = p
-				r.process(t, &wg, r.hp, r.options.protocol, scanopts, output)
+				scanopts.RequestURI = requestURI + p
+				r.process(t, &wg, r.hp, protocol, scanopts, output)
 				reqs++
 			}
 		} else {
-			r.process(t, &wg, r.hp, r.options.protocol, &r.scanopts, output)
+			scanopts := r.scanopts.Clone()
+			scanopts.RequestURI = requestURI + scanopts.RequestURI
+			r.process(t, &wg, r.hp, protocol, scanopts, output)
 			reqs++
 		}
 
@@ -477,10 +496,10 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 			}
 		}
 
-		// the host name shouldn't have any semicolon - in case remove the port
-		semicolonPosition := strings.LastIndex(target, ":")
-		if semicolonPosition > 0 {
-			target = target[:semicolonPosition]
+		// the host name shouldn't have any semicolon or forward slash - in case remove the port
+		unwantedCharPosition := strings.IndexAny(target, ":/")
+		if unwantedCharPosition > 0 {
+			target = target[:unwantedCharPosition]
 		}
 
 		for port, wantedProtocol := range customport.Ports {
@@ -551,7 +570,6 @@ retry:
 		domain = parts[0]
 		customHost = parts[1]
 	}
-
 	URL := fmt.Sprintf("%s://%s", protocol, domain)
 	if port > 0 {
 		URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
@@ -580,6 +598,8 @@ retry:
 		req.Host = customHost
 	}
 
+	reqURI := req.URL.RequestURI()
+
 	hp.SetCustomHeaders(req, hp.CustomHeaders)
 	if scanopts.RequestBody != "" {
 		req.ContentLength = int64(len(scanopts.RequestBody))
@@ -588,7 +608,7 @@ retry:
 
 	var requestDump []byte
 	if scanopts.Unsafe {
-		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), req.RequestURI, req.Header, req.Body, rawhttp.DefaultOptions)
+		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
 		if err != nil {
 			return Result{URL: URL, err: err}
 		}
@@ -619,11 +639,12 @@ retry:
 	var fullURL string
 
 	if resp.StatusCode >= 0 {
-		if port > 0 {
-			fullURL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, scanopts.RequestURI)
-		} else {
-			fullURL = fmt.Sprintf("%s://%s%s", protocol, domain, scanopts.RequestURI)
-		}
+		fullURL = req.URL.String()
+		// if port > 0 {
+		// 	fullURL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, reqURI)
+		// } else {
+		// 	fullURL = fmt.Sprintf("%s://%s%s", protocol, domain, reqURI)
+		// }
 	}
 
 	builder := &strings.Builder{}
@@ -828,9 +849,9 @@ retry:
 
 	// store responses or chain in directory
 	if scanopts.StoreResponse || scanopts.StoreChain {
-		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)
+		domainFile := fmt.Sprintf("%s%s", domain, reqURI)
 		if port > 0 {
-			domainFile = fmt.Sprintf("%s.%d%s", domain, port, scanopts.RequestURI)
+			domainFile = fmt.Sprintf("%s.%d%s", domain, port, reqURI)
 		}
 		// On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux
 		// Truncating length at 255

From 48010eb8aed869f6fff408785138edd20d24a05d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 24 May 2021 06:50:28 +0000
Subject: [PATCH 14/23] chore(deps): bump
 github.com/projectdiscovery/wappalyzergo

Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.3 to 0.0.4.
- [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases)
- [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.3...v0.0.4)

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b4ca60d..3cf77c6 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/rawhttp v0.0.6
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	github.com/projectdiscovery/wappalyzergo v0.0.3
+	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
 	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
diff --git a/go.sum b/go.sum
index 39ea369..fd24e06 100644
--- a/go.sum
+++ b/go.sum
@@ -126,8 +126,8 @@ github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ks
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/projectdiscovery/wappalyzergo v0.0.3 h1:UwaQyl0vbNx+rqwObzvV+YBECQ2NvWAcY/A1KpJyLCA=
-github.com/projectdiscovery/wappalyzergo v0.0.3/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
+github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
+github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=

From 24f3992c7a6281c1f2a6d15ec6beab158f48aab0 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Mon, 24 May 2021 13:15:24 +0200
Subject: [PATCH 15/23] using helper library

---
 common/httputilz/httputilz.go |  6 +--
 common/stringz/stringz.go     | 34 +++----------
 go.mod                        |  7 ++-
 go.sum                        | 16 +++++-
 runner/runner.go              | 94 ++++++++++-------------------------
 5 files changed, 54 insertions(+), 103 deletions(-)

diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go
index 3a91b9c..8b6367c 100644
--- a/common/httputilz/httputilz.go
+++ b/common/httputilz/httputilz.go
@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http/httputil"
-	"net/url"
 	"strings"
 
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/urlutil"
 )
 
 const (
@@ -74,8 +74,8 @@ func ParseRequest(req string, unsafe bool) (method, path string, headers map[str
 	// Handle case with the full http url in path. In that case,
 	// ignore any host header that we encounter and use the path as request URL
 	if strings.HasPrefix(parts[1], "http") {
-		var parsed *url.URL
-		parsed, err = url.Parse(parts[1])
+		var parsed *urlutil.URL
+		parsed, err = urlutil.Parse(parts[1])
 		if err != nil {
 			err = fmt.Errorf("could not parse request URL: %s", err)
 			return
diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
index 2f6f0dd..f8d0529 100644
--- a/common/stringz/stringz.go
+++ b/common/stringz/stringz.go
@@ -1,12 +1,10 @@
 package stringz
 
 import (
-	"fmt"
-	"net/url"
 	"strconv"
 	"strings"
 
-	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/urlutil"
 )
 
 // TrimProtocol removes the HTTP scheme from an URI
@@ -51,28 +49,10 @@ func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
 // http://foo.com -> http://foo.com:80
 // https://foo.com -> https://foo.com:443
 func AddURLDefaultPort(rawURL string) string {
-	u, err := url.Parse(rawURL)
+	u, err := urlutil.Parse(rawURL)
 	if err != nil {
 		return rawURL
 	}
-	// http://[::]
-	if strings.HasPrefix(u.Host, "[") && strings.HasSuffix(u.Host, "]") {
-		if u.Scheme == httpx.HTTPS {
-			u.Host = fmt.Sprintf("%s:%s", u.Host, "443")
-		} else {
-			u.Host = fmt.Sprintf("%s:%s", u.Host, "80")
-		}
-	}
-	// http://foo.com:81
-	// http://foo.com
-	// http://[::]:80
-	if strings.LastIndexByte(u.Host, ':') == -1 {
-		if u.Scheme == httpx.HTTPS {
-			u.Host = fmt.Sprintf("%s:%s", u.Host, "443")
-		} else {
-			u.Host = fmt.Sprintf("%s:%s", u.Host, "80")
-		}
-	}
 	return u.String()
 }
 
@@ -81,15 +61,13 @@ func AddURLDefaultPort(rawURL string) string {
 // http://foo.com:80 -> http://foo.com
 // https://foo.com:443 -> https://foo.com
 func RemoveURLDefaultPort(rawURL string) string {
-	u, err := url.Parse(rawURL)
+	u, err := urlutil.Parse(rawURL)
 	if err != nil {
 		return rawURL
 	}
-	colon := strings.LastIndexByte(u.Host, ':')
-	if colon != -1 {
-		if (u.Scheme == "https" && u.Host[colon+1:] == "443") || u.Scheme == "http" && u.Host[colon+1:] == "80" {
-			u.Host = u.Host[:colon]
-		}
+
+	if u.Scheme == urlutil.HTTP && u.Port == "80" || u.Scheme == urlutil.HTTPS && u.Port == "443" {
+		u.Port = ""
 	}
 	return u.String()
 }
diff --git a/go.mod b/go.mod
index 3319456..dab9fd2 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.9
+	github.com/miekg/dns v1.1.42 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/cdncheck v0.0.2
 	github.com/projectdiscovery/clistats v0.0.8
@@ -19,9 +20,11 @@ require (
 	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/rawhttp v0.0.6
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	github.com/projectdiscovery/wappalyzergo v0.0.3
+	github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60
+	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
+	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
+	golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 // indirect
 	golang.org/x/text v0.3.6
 )
diff --git a/go.sum b/go.sum
index 72a5071..6da8a21 100644
--- a/go.sum
+++ b/go.sum
@@ -68,6 +68,8 @@ github.com/microcosm-cc/bluemonday v1.0.9/go.mod h1:B2riunDr9benLHghZB7hjIgdwSUz
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
+github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -108,8 +110,6 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
-github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49 h1:hzMhw71p7+a5oyd3si9D/PMtqomvdsrd782+9Y1tUVo=
-github.com/projectdiscovery/httputil v0.0.0-20210506091701-9ad2e8818e49/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
 github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d h1:IdBTOSGaPrZ8+FK0uYMQIva9dYIR5F55PLFWYtBBKc0=
 github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d/go.mod h1:Vm2DY4NwUV5yA6TNzJOOjTYGjTcVfuEN8m9Y5dAksLQ=
 github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE=
@@ -128,8 +128,16 @@ github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ks
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0 h1:HvtSWR3UWX1nuIgkrgCjcdXOH4gTjLVVdZBKu31N/6o=
+github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/urlutil v0.0.0-20210524061804-c77cf80fdec7 h1:dB9yoASdWITOq670Zr+ziLBjn8TWN1VQV7cACFmm2tU=
+github.com/projectdiscovery/urlutil v0.0.0-20210524061804-c77cf80fdec7/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
+github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60 h1:pCMNCJM+XHKGiILvsfZ3Wp3fSOhEr36qu64HJBVGyNk=
+github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/wappalyzergo v0.0.3 h1:UwaQyl0vbNx+rqwObzvV+YBECQ2NvWAcY/A1KpJyLCA=
 github.com/projectdiscovery/wappalyzergo v0.0.3/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
+github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
+github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
@@ -174,6 +182,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -201,6 +211,8 @@ golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 h1:lCnv+lfrU9FRPGf8NeRuWAAPjNnema5WtBinMgs1fD8=
+golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
diff --git a/runner/runner.go b/runner/runner.go
index c55d3b7..a4a9448 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
-	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -22,6 +21,8 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/clistats"
+	"github.com/projectdiscovery/urlutil"
+
 	// automatic fd max increase if running as root
 	_ "github.com/projectdiscovery/fdmax/autofdmax"
 	"github.com/projectdiscovery/gologger"
@@ -412,39 +413,20 @@ func (r *Runner) RunEnumeration() {
 		t := string(k)
 		var reqs int
 		protocol := r.options.protocol
-		requestURI := ""
 		// attempt to parse url as is
-		if u, err := url.Parse(t); err == nil {
-			switch u.Scheme {
-			case httpx.HTTP:
-				protocol = httpx.HTTP
-			case httpx.HTTPS:
-				protocol = httpx.HTTPS
-			}
-			requestURI = u.RequestURI()
-			if !strings.HasPrefix(requestURI, "/") {
-				// if requestURI doesn't start with "/" attempts to reparse with generic http protocol
-				if u, err := url.Parse("http://" + t); err == nil {
-					requestURI = u.RequestURI()
-				}
-			}
-			// if it's only "/" skip it
-			if requestURI == "/" {
-				requestURI = ""
-			}
+		if u, err := urlutil.Parse(t); err == nil {
+			protocol = u.Scheme
 		}
 
 		if len(r.options.requestURIs) > 0 {
 			for _, p := range r.options.requestURIs {
 				scanopts := r.scanopts.Clone()
-				scanopts.RequestURI = requestURI + p
+				scanopts.RequestURI = p
 				r.process(t, &wg, r.hp, protocol, scanopts, output)
 				reqs++
 			}
 		} else {
-			scanopts := r.scanopts.Clone()
-			scanopts.RequestURI = requestURI + scanopts.RequestURI
-			r.process(t, &wg, r.hp, protocol, scanopts, output)
+			r.process(t, &wg, r.hp, protocol, &r.scanopts, output)
 			reqs++
 		}
 
@@ -474,7 +456,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 					wg.Add()
 					go func(target, method, protocol string) {
 						defer wg.Done()
-						result := r.analyze(hp, protocol, target, 0, method, scanopts)
+						result := r.analyze(hp, protocol, target, method, scanopts)
 						output <- result
 						if scanopts.TLSProbe && result.TLSData != nil {
 							scanopts.TLSProbe = false
@@ -496,18 +478,13 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 			}
 		}
 
-		// the host name shouldn't have any semicolon or forward slash - in case remove the port
-		unwantedCharPosition := strings.IndexAny(target, ":/")
-		if unwantedCharPosition > 0 {
-			target = target[:unwantedCharPosition]
-		}
-
 		for port, wantedProtocol := range customport.Ports {
 			for _, method := range scanopts.Methods {
 				wg.Add()
 				go func(port int, method, protocol string) {
 					defer wg.Done()
-					result := r.analyze(hp, protocol, target, port, method, scanopts)
+					target, _ := urlutil.ChangePort(target, fmt.Sprint(port))
+					result := r.analyze(hp, protocol, target, method, scanopts)
 					output <- result
 					if scanopts.TLSProbe && result.TLSData != nil {
 						scanopts.TLSProbe = false
@@ -553,7 +530,7 @@ func targets(target string) chan string {
 	return results
 }
 
-func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, port int, method string, scanopts *scanOptions) Result {
+func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, method string, scanopts *scanOptions) Result {
 	origProtocol := protocol
 	if protocol == httpx.HTTPorHTTPS {
 		protocol = httpx.HTTPS
@@ -570,29 +547,16 @@ retry:
 		domain = parts[0]
 		customHost = parts[1]
 	}
-	URL := fmt.Sprintf("%s://%s", protocol, domain)
-	if port > 0 {
-		URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
-	} else {
-		domainParse := strings.Split(domain, ":")
-		domain = domainParse[0]
-		if len(domainParse) > 1 {
-			// consider the port till the next /
-			if strings.Contains(domainParse[1], "/") {
-				iForwardSlash := strings.Index(domainParse[1], "/")
-				domainParse[1] = domainParse[1][:iForwardSlash]
-			}
-			port, _ = strconv.Atoi(domainParse[1])
-		}
-	}
+	URL, _ := urlutil.Parse(domain)
+	URL.Scheme = protocol
 
 	if !scanopts.Unsafe {
-		URL += scanopts.RequestURI
+		URL.RequestURI += scanopts.RequestURI
 	}
 
-	req, err := hp.NewRequest(method, URL)
+	req, err := hp.NewRequest(method, URL.String())
 	if err != nil {
-		return Result{URL: URL, err: err}
+		return Result{URL: URL.String(), err: err}
 	}
 	if customHost != "" {
 		req.Host = customHost
@@ -610,7 +574,7 @@ retry:
 	if scanopts.Unsafe {
 		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
 		if err != nil {
-			return Result{URL: URL, err: err}
+			return Result{URL: URL.String(), err: err}
 		}
 	} else {
 		// Create a copy on the fly of the request body - ignore errors
@@ -618,7 +582,7 @@ retry:
 		req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 		requestDump, err = httputil.DumpRequestOut(req.Request, true)
 		if err != nil {
-			return Result{URL: URL, err: err}
+			return Result{URL: URL.String(), err: err}
 		}
 	}
 
@@ -633,18 +597,13 @@ retry:
 			retried = true
 			goto retry
 		}
-		return Result{URL: URL, err: err}
+		return Result{URL: URL.String(), err: err}
 	}
 
 	var fullURL string
 
 	if resp.StatusCode >= 0 {
 		fullURL = req.URL.String()
-		// if port > 0 {
-		// 	fullURL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, reqURI)
-		// } else {
-		// 	fullURL = fmt.Sprintf("%s://%s%s", protocol, domain, reqURI)
-		// }
 	}
 
 	builder := &strings.Builder{}
@@ -758,6 +717,7 @@ retry:
 
 	pipeline := false
 	if scanopts.Pipeline {
+		port, _ := strconv.Atoi(URL.Port)
 		pipeline = hp.SupportPipeline(protocol, method, domain, port)
 		if pipeline {
 			builder.WriteString(" [pipeline]")
@@ -767,7 +727,7 @@ retry:
 	var http2 bool
 	// if requested probes for http2
 	if scanopts.HTTP2Probe {
-		http2 = hp.SupportHTTP2(protocol, method, URL)
+		http2 = hp.SupportHTTP2(protocol, method, URL.String())
 		if http2 {
 			builder.WriteString(" [http2]")
 		}
@@ -849,10 +809,8 @@ retry:
 
 	// store responses or chain in directory
 	if scanopts.StoreResponse || scanopts.StoreChain {
-		domainFile := fmt.Sprintf("%s%s", domain, reqURI)
-		if port > 0 {
-			domainFile = fmt.Sprintf("%s.%d%s", domain, port, reqURI)
-		}
+		domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
+
 		// On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux
 		// Truncating length at 255
 		if len(domainFile) >= maxFileNameLength {
@@ -881,12 +839,12 @@ retry:
 		}
 	}
 
-	parsed, err := url.Parse(fullURL)
+	parsed, err := urlutil.Parse(fullURL)
 	if err != nil {
-		return Result{URL: URL, err: errors.Wrap(err, "could not parse url")}
+		return Result{URL: fullURL, err: errors.Wrap(err, "could not parse url")}
 	}
 
-	finalPort := parsed.Port()
+	finalPort := parsed.Port
 	if finalPort == "" {
 		if parsed.Scheme == "http" {
 			finalPort = "80"
@@ -894,7 +852,7 @@ retry:
 			finalPort = "443"
 		}
 	}
-	finalPath := parsed.Path
+	finalPath := parsed.RequestURI
 	if finalPath == "" {
 		finalPath = "/"
 	}

From 3e9802a776895b5f6dacf6944c2fe8ad8811c07b Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Tue, 25 May 2021 15:56:09 +0200
Subject: [PATCH 16/23] updating deps

---
 go.mod |  2 +-
 go.sum | 11 ++---------
 2 files changed, 3 insertions(+), 10 deletions(-)

diff --git a/go.mod b/go.mod
index dab9fd2..4af6e1d 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,7 @@ require (
 	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
-	golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 // indirect
 	golang.org/x/text v0.3.6
 )
diff --git a/go.sum b/go.sum
index 6da8a21..0a4341a 100644
--- a/go.sum
+++ b/go.sum
@@ -66,7 +66,6 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z
 github.com/microcosm-cc/bluemonday v1.0.9 h1:dpCwruVKoyrULicJwhuY76jB+nIxRVKv/e248Vx/BXg=
 github.com/microcosm-cc/bluemonday v1.0.9/go.mod h1:B2riunDr9benLHghZB7hjIgdwSUzzs0pjCxFrWYEZFU=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -130,12 +129,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0 h1:HvtSWR3UWX1nuIgkrgCjcdXOH4gTjLVVdZBKu31N/6o=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/urlutil v0.0.0-20210524061804-c77cf80fdec7 h1:dB9yoASdWITOq670Zr+ziLBjn8TWN1VQV7cACFmm2tU=
-github.com/projectdiscovery/urlutil v0.0.0-20210524061804-c77cf80fdec7/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60 h1:pCMNCJM+XHKGiILvsfZ3Wp3fSOhEr36qu64HJBVGyNk=
 github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
-github.com/projectdiscovery/wappalyzergo v0.0.3 h1:UwaQyl0vbNx+rqwObzvV+YBECQ2NvWAcY/A1KpJyLCA=
-github.com/projectdiscovery/wappalyzergo v0.0.3/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
 github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -180,10 +175,9 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
-golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -209,7 +203,6 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 h1:lCnv+lfrU9FRPGf8NeRuWAAPjNnema5WtBinMgs1fD8=
 golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

From d4c23f5342a22cc6ed57dddf0443299a40c999c6 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 25 May 2021 16:11:30 +0200
Subject: [PATCH 17/23] golint => revive

---
 .golangci.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index 1a2c5d3..dc27d7a 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -23,8 +23,6 @@ linters-settings:
   #   min-complexity: 15
   goimports:
     local-prefixes: github.com/golangci/golangci-lint
-  golint:
-    min-confidence: 0
   gomnd:
     settings:
       mnd:
@@ -67,7 +65,6 @@ linters:
     - gocritic
     - gofmt
     - goimports
-    - golint
     - gomnd
     - goprintffuncname
     - gosimple
@@ -88,6 +85,7 @@ linters:
     - unused
     - varcheck
     - whitespace
+    - revive
 
   # don't enable:
   # - depguard

From 46f0b1662a3bafda36e064fa292316a485c67bfd Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 25 May 2021 16:11:40 +0200
Subject: [PATCH 18/23] misc

---
 runner/runner.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runner/runner.go b/runner/runner.go
index a4a9448..96f66d2 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -483,8 +483,8 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 				wg.Add()
 				go func(port int, method, protocol string) {
 					defer wg.Done()
-					target, _ := urlutil.ChangePort(target, fmt.Sprint(port))
-					result := r.analyze(hp, protocol, target, method, scanopts)
+					h, _ := urlutil.ChangePort(target, fmt.Sprint(port))
+					result := r.analyze(hp, protocol, h, method, scanopts)
 					output <- result
 					if scanopts.TLSProbe && result.TLSData != nil {
 						scanopts.TLSProbe = false
@@ -530,7 +530,7 @@ func targets(target string) chan string {
 	return results
 }
 
-func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain string, method string, scanopts *scanOptions) Result {
+func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method string, scanopts *scanOptions) Result {
 	origProtocol := protocol
 	if protocol == httpx.HTTPorHTTPS {
 		protocol = httpx.HTTPS

From f623a21564e23eb8f696b06b9ec3807ed3d25b8b Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 25 May 2021 16:23:32 +0200
Subject: [PATCH 19/23] removing deprecated golint directive

---
 .golangci.yml         | 7 -------
 common/httpx/csp.go   | 2 +-
 common/httpx/title.go | 6 +++---
 go.mod                | 3 ++-
 go.sum                | 7 ++++---
 5 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index dc27d7a..dd00467 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -109,10 +109,3 @@ issues:
   exclude:
     # should have a package comment, unless it's in another file for this package (golint)
     - 'in another file for this package'
-
-# golangci.com configuration
-# https://github.com/golangci/golangci/wiki/Configuration
-service:
-  golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
-  prepare:
-    - echo "here I can run custom commands, but no preparation needed for this repo"
diff --git a/common/httpx/csp.go b/common/httpx/csp.go
index 0640d70..7a68ac9 100644
--- a/common/httpx/csp.go
+++ b/common/httpx/csp.go
@@ -8,7 +8,7 @@ import (
 )
 
 // CSPHeaders is an incomplete list of most common CSP headers
-var CSPHeaders []string = []string{
+var CSPHeaders = []string{
 	"Content-Security-Policy",               // standard
 	"Content-Security-Policy-Report-Only",   // standard
 	"X-Content-Security-Policy-Report-Only", // non - standard
diff --git a/common/httpx/title.go b/common/httpx/title.go
index c80aefa..1edb534 100644
--- a/common/httpx/title.go
+++ b/common/httpx/title.go
@@ -11,9 +11,9 @@ import (
 )
 
 var (
-	cutset                       = "\n\t\v\f\r"
-	reTitle       *regexp.Regexp = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
-	reContentType *regexp.Regexp = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	cutset        = "\n\t\v\f\r"
+	reTitle       = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
+	reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
 )
 
 // ExtractTitle from a response
diff --git a/go.mod b/go.mod
index 4af6e1d..9ba039e 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,8 @@ require (
 	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/rawhttp v0.0.6
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60
+	github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda // indirect
+	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
 	github.com/projectdiscovery/wappalyzergo v0.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
diff --git a/go.sum b/go.sum
index 0a4341a..98cf93c 100644
--- a/go.sum
+++ b/go.sum
@@ -127,10 +127,11 @@ github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ks
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
-github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0 h1:HvtSWR3UWX1nuIgkrgCjcdXOH4gTjLVVdZBKu31N/6o=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
-github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60 h1:pCMNCJM+XHKGiILvsfZ3Wp3fSOhEr36qu64HJBVGyNk=
-github.com/projectdiscovery/urlutil v0.0.0-20210524085509-c534bc15ea60/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
+github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda h1:q9o7dHZ22CGUWUAzp26nYoCugOkwBDdqMfgMDpIrakM=
+github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
+github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo=
 github.com/projectdiscovery/wappalyzergo v0.0.4 h1:JEgo9JzpOc9zdF0RMt3esz4yap4+SQ5WFr81CfPHA84=
 github.com/projectdiscovery/wappalyzergo v0.0.4/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

From 08a4fdf0210aa5a8e1001b318a6f21ef82dfbc53 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 25 May 2021 16:29:32 +0200
Subject: [PATCH 20/23] updating golint version

---
 .github/workflows/build.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 99bf43b..749babe 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -12,10 +12,10 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2.5.2
+        uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.31
+          version: latest
           args: --timeout 5m
 
   build:

From 490c7cc67be266f3c40e1ba897ffa36b5a3d6fb2 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Tue, 25 May 2021 19:24:04 +0200
Subject: [PATCH 21/23] Updating retryablehttp with http2 support

Closes #274
---
 go.mod | 2 +-
 go.sum | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 9ba039e..eea382b 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46
 	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/rawhttp v0.0.6
-	github.com/projectdiscovery/retryablehttp-go v1.0.1
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
 	github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda // indirect
 	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
 	github.com/projectdiscovery/wappalyzergo v0.0.4
diff --git a/go.sum b/go.sum
index 98cf93c..623b3a6 100644
--- a/go.sum
+++ b/go.sum
@@ -125,8 +125,9 @@ github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8
 github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
 github.com/projectdiscovery/retryabledns v1.0.11 h1:jyzTass/CD3MgaK4pQSXJzwb91ksVYocwiE0AQ1ytEo=
 github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
-github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 h1:CJHP3CLCc/eqdXQEvZy8KiiqtAk9kEsd1URtPyPAQ1s=
+github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda h1:q9o7dHZ22CGUWUAzp26nYoCugOkwBDdqMfgMDpIrakM=
 github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
@@ -177,6 +178,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

From 94c5ebede32705192834eefc3cd2e45775ce1843 Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 26 May 2021 02:07:38 +0200
Subject: [PATCH 22/23] Fixing body behavior causing 307/308 failure

---
 runner/runner.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/runner/runner.go b/runner/runner.go
index 96f66d2..db616b3 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -565,11 +565,14 @@ retry:
 	reqURI := req.URL.RequestURI()
 
 	hp.SetCustomHeaders(req, hp.CustomHeaders)
+	// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
 	if scanopts.RequestBody != "" {
 		req.ContentLength = int64(len(scanopts.RequestBody))
 		req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody))
+	} else {
+		req.ContentLength = 0
+		req.Body = nil
 	}
-
 	var requestDump []byte
 	if scanopts.Unsafe {
 		requestDump, err = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
@@ -577,13 +580,18 @@ retry:
 			return Result{URL: URL.String(), err: err}
 		}
 	} else {
-		// Create a copy on the fly of the request body - ignore errors
+		// Create a copy on the fly of the request body
 		bodyBytes, _ := req.BodyBytes()
 		req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 		requestDump, err = httputil.DumpRequestOut(req.Request, true)
 		if err != nil {
 			return Result{URL: URL.String(), err: err}
 		}
+		// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
+		// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
+		if len(bodyBytes) <= 0 {
+			req.Body = nil
+		}
 	}
 
 	resp, err := hp.Do(req)

From 8aebd646c5f08fe4d075a8d153d898e7427a774a Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Wed, 26 May 2021 02:09:34 +0200
Subject: [PATCH 23/23] misc

---
 runner/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runner/runner.go b/runner/runner.go
index db616b3..2ce477b 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -589,7 +589,7 @@ retry:
 		}
 		// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
 		// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
-		if len(bodyBytes) <= 0 {
+		if len(bodyBytes) == 0 {
 			req.Body = nil
 		}
 	}