Merge pull request #46 from projectdiscovery/37-bugfix

fixed automatic decoding

common/httpx/encodings.go

@@ -0,0 +1,45 @@
+package httpx
+
+import (
+	"bytes"
+	"io/ioutil"
+
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/encoding/traditionalchinese"
+	"golang.org/x/text/transform"
+)
+
+// Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5
+
+//convert GBK to UTF-8
+func Decodegbk(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}
+
+//convert BIG5 to UTF-8
+func Decodebig5(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}
+
+//convert UTF-8 to BIG5
+func Encodebig5(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}

common/httpx/httpx.go

@@ -172,6 +172,8 @@ func (h *HTTPX) NewRequest(method, URL string) (req *retryablehttp.Request, err
 
 	// set default user agent
 	req.Header.Set("User-Agent", h.Options.DefaultUserAgent)
+	// set default encoding to accept utf8
+	req.Header.Add("Accept-Charset", "utf-8")
 	return
 }
 

common/httpx/title.go

@@ -8,12 +8,29 @@ import (
 )
 
 // ExtractTitle from a response
-func ExtractTitle(r *Response) string {
+func ExtractTitle(r *Response) (title string) {
 	var re = regexp.MustCompile(`(?im)<\s*title *>(.*?)<\s*/\s*title>`)
 	for _, match := range re.FindAllString(r.Raw, -1) {
-		return html.UnescapeString(trimTitleTags(match))
+		title = html.UnescapeString(trimTitleTags(match))
+		break
 	}
-	return ""
+
+	// Non UTF-8
+	if contentTypes, ok := r.Headers["Content-Type"]; ok {
+		contentType := strings.Join(contentTypes, ";")
+
+		// special cases
+		if strings.Contains(contentType, "charset=GB2312") {
+			titleUtf8, err := Decodegbk([]byte(title))
+			if err != nil {
+				return
+			}
+
+			return string(titleUtf8)
+		}
+	}
+
+	return
 }
 
 func trimTitleTags(title string) string {

go.mod

@@ -14,4 +14,5 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.2.1
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
+	golang.org/x/text v0.3.0
 )

go.sum

@@ -46,6 +46,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=