projectdiscovery-httpx/common/httpx/csp.go

package httpx

import (
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	mapsutil "github.com/projectdiscovery/utils/maps"
	stringsutil "github.com/projectdiscovery/utils/strings"
	"github.com/weppos/publicsuffix-go/publicsuffix"
)

// CSPHeaders is an incomplete list of most common CSP headers
var CSPHeaders = []string{
	"Content-Security-Policy",               // standard
	"Content-Security-Policy-Report-Only",   // standard
	"X-Content-Security-Policy-Report-Only", // non - standard
	"X-Webkit-Csp-Report-Only",              // non - standard
}

// CSPData contains the Content-Security-Policy domain list
type CSPData struct {
	Fqdns   []string `json:"fqdn,omitempty"`
	Domains []string `json:"domains,omitempty"`
}

// CSPGrab fills the CSPData
func (h *HTTPX) CSPGrab(r *Response) *CSPData {
	domains := make(map[string]struct{})
	fqdns := make(map[string]struct{})
	// extract from headers
	for _, cspHeader := range CSPHeaders {
		if cspValues, ok := r.Headers[cspHeader]; ok {
			for _, cspValue := range cspValues {
				parsePotentialDomains(fqdns, domains, cspValue)
			}
		}
	}

	// extract from body
	if len(r.Data) > 0 {
		doc, err := goquery.NewDocumentFromReader(bytes.NewReader(r.Data))
		if err == nil {
			doc.Find("meta").Each(func(i int, s *goquery.Selection) {
				if _, ok := s.Attr("http-equiv"); ok {
					if content, ok := s.Attr("content"); ok {
						parsePotentialDomains(fqdns, domains, content)
					}
				}
			})
		}
	}

	if len(domains) > 0 || len(fqdns) > 0 {
		return &CSPData{Domains: mapsutil.GetKeys(domains), Fqdns: mapsutil.GetKeys(fqdns)}
	}
	return nil
}

func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
	// rule is like aa bb domain1 domain2 domain3
	tokens := stringsutil.SplitAny(data, " ", ";", ",")
	// we extracts only potential domains
	for _, t := range tokens {
		if isPotentialDomain(t) {
			if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
				domains[dn.SLD+"."+dn.TLD] = struct{}{}
				if dn.TRD != "" {
					fqdns[dn.String()] = struct{}{}
				}
			}
		}
	}
}

func isPotentialDomain(s string) bool {
	return strings.Contains(s, ".") || strings.HasPrefix(s, "http")
}

func extractDomain(str string) string {
	str = removeWildcards(str)
	u := str
	if !strings.Contains(str, "://") {
		u = "https://" + str
	}
	u = sanitizeURL(u)
	parsedURL, err := url.Parse(u)
	if err != nil {
		return ""
	}
	return parsedURL.Hostname()
}

func removeWildcards(domain string) string {
	if stringsutil.HasPrefixAny(domain, "'", "\"") {
		domain = domain[1:]
	}
	if stringsutil.HasSuffixAny(domain, "'", "\"") {
		domain = domain[:len(domain)-1]
	}
	if strings.Contains(domain, "://") {
		domain = strings.Split(domain, "://")[1]
	}
	parts := []string{}
	for _, part := range strings.Split(domain, ".") {
		if !strings.Contains(part, "*") {
			parts = append(parts, part)
		}
	}
	return strings.Join(parts, ".")
}

var urlInvalidCharRegex = regexp.MustCompile(`[^\w-./:~]`)

func sanitizeURL(u string) string {
	// Replace invalid characters with percent-encoded equivalents
	return urlInvalidCharRegex.ReplaceAllStringFunc(u, func(match string) string {
		return fmt.Sprintf("%%%02X", match[0])
	})
}