Add fqdn to csp output

common/httpx/csp.go

@@ -2,11 +2,13 @@ package httpx
 
 import (
 	"bytes"
+	"net/url"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	"github.com/weppos/publicsuffix-go/publicsuffix"
 )
 
 // CSPHeaders is an incomplete list of most common CSP headers
@@ -19,17 +21,19 @@ var CSPHeaders = []string{
 
 // CSPData contains the Content-Security-Policy domain list
 type CSPData struct {
+	Fqdns   []string `json:"fqdns,omitempty"`
 	Domains []string `json:"domains,omitempty"`
 }
 
 // CSPGrab fills the CSPData
 func (h *HTTPX) CSPGrab(r *Response) *CSPData {
 	domains := make(map[string]struct{})
+	fqdns := make(map[string]struct{})
 	// extract from headers
 	for _, cspHeader := range CSPHeaders {
 		if cspValues, ok := r.Headers[cspHeader]; ok {
 			for _, cspValue := range cspValues {
-				parsePotentialDomains(domains, cspValue)
+				parsePotentialDomains(fqdns, domains, cspValue)
 			}
 		}
 	}
@@ -41,26 +45,29 @@ func (h *HTTPX) CSPGrab(r *Response) *CSPData {
 			doc.Find("meta").Each(func(i int, s *goquery.Selection) {
 				if _, ok := s.Attr("http-equiv"); ok {
 					if content, ok := s.Attr("content"); ok {
-						parsePotentialDomains(domains, content)
+						parsePotentialDomains(fqdns, domains, content)
 					}
 				}
 			})
 		}
 	}
 
-	if len(domains) > 0 {
-		return &CSPData{Domains: mapsutil.GetKeys(domains)}
+	if len(domains) > 0 || len(fqdns) > 0 {
+		return &CSPData{Domains: mapsutil.GetKeys(domains), Fqdns: mapsutil.GetKeys(fqdns)}
 	}
 	return nil
 }
 
-func parsePotentialDomains(domains map[string]struct{}, data string) {
+func parsePotentialDomains(fqdns, domains map[string]struct{}, data string) {
 	// rule is like aa bb domain1 domain2 domain3
 	tokens := stringsutil.SplitAny(data, " ", ";", ",")
 	// we extracts only potential domains
 	for _, t := range tokens {
 		if isPotentialDomain(t) {
-			domains[t] = struct{}{}
+			if dn, err := publicsuffix.Parse(extractDomain(t)); err == nil {
+				domains[dn.SLD+"."+dn.TLD] = struct{}{}
+				fqdns[dn.String()] = struct{}{}
+			}
 		}
 	}
 }
@@ -68,3 +75,15 @@ func parsePotentialDomains(domains map[string]struct{}, data string) {
 func isPotentialDomain(s string) bool {
 	return strings.Contains(s, ".") || strings.HasPrefix(s, "http")
 }
+
+func extractDomain(str string) string {
+	u := str
+	if !strings.Contains(str, "://") {
+		u = "https://" + str
+	}
+	parsedURL, err := url.Parse(u)
+	if err != nil {
+		return str
+	}
+	return parsedURL.Host
+}