projectdiscovery-httpx/common/stringz/stringz.go

package stringz

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"mime"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	stringsutil "github.com/projectdiscovery/utils/strings"
	urlutil "github.com/projectdiscovery/utils/url"
	"github.com/spaolacci/murmur3"
)

// TrimProtocol removes the HTTP scheme from an URI
func TrimProtocol(targetURL string, addDefaultPort bool) string {
	URL := strings.TrimSpace(targetURL)
	if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") {
		if addDefaultPort {
			URL = AddURLDefaultPort(URL)
			URL = URL[strings.Index(URL, "//")+2:]
		}
	}

	return URL
}

// StringToSliceInt converts string to slice of ints
func StringToSliceInt(s string) ([]int, error) {
	var r []int
	if s == "" {
		return r, nil
	}
	for _, v := range strings.Split(s, ",") {
		vTrim := strings.TrimSpace(v)
		if i, err := strconv.Atoi(vTrim); err == nil {
			r = append(r, i)
		} else {
			return r, err
		}
	}

	return r, nil
}

// StringToSliceUInt converts string to slice of ints
func StringToSliceUInt32(s string) ([]uint32, error) {
	var r []uint32
	if s == "" {
		return r, nil
	}
	for _, v := range strings.Split(s, ",") {
		vTrim := strings.TrimSpace(v)
		if i, err := strconv.ParseUint(vTrim, 10, 64); err == nil {
			r = append(r, uint32(i))
		} else {
			return r, err
		}
	}

	return r, nil
}

// SplitByCharAndTrimSpace splits string by a character and remove spaces
func SplitByCharAndTrimSpace(s, splitchar string) (result []string) {
	for _, token := range strings.Split(s, splitchar) {
		result = append(result, strings.TrimSpace(token))
	}
	return
}

// AddURLDefaultPort add url default port (80/443) from an URI
// eg:
// http://foo.com -> http://foo.com:80
// https://foo.com -> https://foo.com:443
func AddURLDefaultPort(rawURL string) string {
	u, err := urlutil.Parse(rawURL)
	if err != nil {
		return rawURL
	}
	// Force default port to be added if not present
	if u.Port() == "" {
		switch u.Scheme {
		case urlutil.HTTP:
			u.UpdatePort("80")
		case urlutil.HTTPS:
			u.UpdatePort("443")
		}
	}
	return u.String()
}

// RemoveURLDefaultPort remove url default port (80/443) from an URI
// eg:
// http://foo.com:80 -> http://foo.com
// https://foo.com:443 -> https://foo.com
func RemoveURLDefaultPort(rawURL string) string {
	u, err := urlutil.Parse(rawURL)
	if err != nil {
		return rawURL
	}

	if u.Scheme == urlutil.HTTP && u.Port() == "80" || u.Scheme == urlutil.HTTPS && u.Port() == "443" {
		u.TrimPort()
	}
	return u.String()
}

func GetInvalidURI(rawURL string) (bool, string) {
	if _, err := url.Parse(rawURL); err != nil {
		if u, err := urlutil.Parse(rawURL); err == nil {
			return true, u.GetRelativePath()
		}
	}
	return false, ""
}

func isContentTypeImage(data []byte) bool {
	contentType := http.DetectContentType(data)
	return stringsutil.HasPrefixAny(contentType, "image/")
}

func murmurhash(data []byte) int32 {
	stdBase64 := base64.StdEncoding.EncodeToString(data)
	stdBase64 = InsertInto(stdBase64, 76, '\n')
	hasher := murmur3.New32WithSeed(0)
	hasher.Write([]byte(stdBase64))
	return int32(hasher.Sum32())
}

// md5Hash returns the md5 hash of the data
func md5Hash(data []byte) string {
	hasher := md5.New()
	hasher.Write(data)
	return hex.EncodeToString(hasher.Sum(nil))
}

func FaviconHash(data []byte) (int32, string, error) {
	if isContentTypeImage(data) {
		return murmurhash(data), md5Hash(data), nil
	}

	return 0, "", errors.New("content type is not image")
}

func InsertInto(s string, interval int, sep rune) string {
	var buffer bytes.Buffer
	before := interval - 1
	last := len(s) - 1
	for i, char := range s {
		buffer.WriteRune(char)
		if i%interval == before && i != last {
			buffer.WriteRune(sep)
		}
	}
	buffer.WriteRune(sep)
	return buffer.String()
}

// Base64 returns base64 of given byte array
func Base64(bin []byte) string {
	return base64.StdEncoding.EncodeToString(bin)
}

func IsBase64Icon(iconBase64 string) bool {
	if iconBase64 == "" {
		return false
	}
	parts := strings.Split(strings.TrimPrefix(iconBase64, "data:"), ",")
	if len(parts) != 2 {
		return false
	}
	mediaType, _, _ := mime.ParseMediaType(parts[0])
	return strings.HasPrefix(mediaType, "image/")
}

func DecodeBase64Icon(iconBase64 string) ([]byte, error) {
	if iconBase64 == "" {
		return nil, errors.New("empty base64 icon")
	}
	parts := strings.Split(iconBase64, ",")
	if len(parts) != 2 {
		return nil, errors.New("invalid base64 icon")
	}
	return base64.StdEncoding.DecodeString(strings.TrimSpace(parts[1]))
}