diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go new file mode 100644 index 0000000..4adcc8f --- /dev/null +++ b/common/httpx/encodings.go @@ -0,0 +1,45 @@ +package httpx + +import ( + "bytes" + "io/ioutil" + + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/transform" +) + +// Credits: https://gist.github.com/zhangbaohe/c691e1da5bbdc7f41ca5 + +//convert GBK to UTF-8 +func Decodegbk(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} + +//convert BIG5 to UTF-8 +func Decodebig5(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} + +//convert UTF-8 to BIG5 +func Encodebig5(s []byte) ([]byte, error) { + I := bytes.NewReader(s) + O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder()) + d, e := ioutil.ReadAll(O) + if e != nil { + return nil, e + } + return d, nil +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index dfb150e..e81b2fb 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -172,6 +172,8 @@ func (h *HTTPX) NewRequest(method, URL string) (req *retryablehttp.Request, err // set default user agent req.Header.Set("User-Agent", h.Options.DefaultUserAgent) + // set default encoding to accept utf8 + req.Header.Add("Accept-Charset", "utf-8") return } diff --git a/common/httpx/title.go b/common/httpx/title.go index e505112..be7ef4e 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -8,12 +8,29 @@ import ( ) // ExtractTitle from a response -func ExtractTitle(r *Response) string { +func ExtractTitle(r *Response) (title string) { var re = regexp.MustCompile(`(?im)<\s*title *>(.*?)<\s*/\s*title>`) for _, match := range re.FindAllString(r.Raw, -1) { - return html.UnescapeString(trimTitleTags(match)) + title = html.UnescapeString(trimTitleTags(match)) + break } - return "" + + // Non UTF-8 + if contentTypes, ok := r.Headers["Content-Type"]; ok { + contentType := strings.Join(contentTypes, ";") + + // special cases + if strings.Contains(contentType, "charset=GB2312") { + titleUtf8, err := Decodegbk([]byte(title)) + if err != nil { + return + } + + return string(titleUtf8) + } + } + + return } func trimTitleTags(title string) string { diff --git a/go.mod b/go.mod index 2a7cda2..0b2f3b2 100644 --- a/go.mod +++ b/go.mod @@ -14,4 +14,5 @@ require ( github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.2.1 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 + golang.org/x/text v0.3.0 ) diff --git a/go.sum b/go.sum index 5ca273a..77a0afd 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=