From ca20e5538c7ae993bc0d58fbd16e24f08015390c Mon Sep 17 00:00:00 2001
From: Mzack9999 <mzack9999@protonmail.com>
Date: Mon, 14 Sep 2020 02:06:35 +0200
Subject: [PATCH] extended CSP extraction

---
 common/httpx/csp.go   | 40 +++++++++++++++++++++++++++-------------
 common/slice/slice.go |  9 +++++++++
 go.mod                |  2 +-
 go.sum                |  6 ++----
 4 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/common/httpx/csp.go b/common/httpx/csp.go
index a2dbc2d..6af4be7 100644
--- a/common/httpx/csp.go
+++ b/common/httpx/csp.go
@@ -3,33 +3,47 @@ package httpx
 import (
 	"net/http"
 	"strings"
+
+	"github.com/projectdiscovery/httpx/common/slice"
 )
 
+// CSPHeaders is an incomplete list of most common CSP headers
+var CSPHeaders []string = []string{
+	"Content-Security-Policy",               //standard
+	"Content-Security-Policy-Report-Only",   //standard
+	"X-Content-Security-Policy-Report-Only", // non - standard
+	"X-Webkit-Csp-Report-Only",              // non - standard
+}
+
 type CspData struct {
 	Domains []string `json:"domains,omitempty"`
 }
 
 func (h *HTTPX) CspGrab(r *http.Response) *CspData {
-	cspRaw := r.Header.Get("Content-Security-Policy")
-	if cspRaw != "" {
-		var domains []string
-		rules := strings.Split(cspRaw, ";")
-		for _, rule := range rules {
-			// rule is like aa bb domain1 domain2 domain3
-			tokens := strings.Split(rule, " ")
-			// we extracts only potential domains
-			for _, t := range tokens {
-				if isPotentialDomain(t) {
-					domains = append(domains, t)
+	domains := make(map[string]struct{})
+	for _, cspHeader := range CSPHeaders {
+		cspRaw := r.Header.Get(cspHeader)
+		if cspRaw != "" {
+			rules := strings.Split(cspRaw, ";")
+			for _, rule := range rules {
+				// rule is like aa bb domain1 domain2 domain3
+				tokens := strings.Split(rule, " ")
+				// we extracts only potential domains
+				for _, t := range tokens {
+					if isPotentialDomain(t) {
+						domains[t] = struct{}{}
+					}
 				}
 			}
 		}
-		return &CspData{Domains: domains}
+	}
+
+	if len(domains) > 0 {
+		return &CspData{Domains: slice.ToSlice(domains)}
 	}
 	return nil
 }
 
-// bare minimum conditions to filter potential domains
 func isPotentialDomain(s string) bool {
 	return strings.Contains(s, ".") || strings.HasPrefix(s, "http")
 }
diff --git a/common/slice/slice.go b/common/slice/slice.go
index fc62339..0102490 100644
--- a/common/slice/slice.go
+++ b/common/slice/slice.go
@@ -9,3 +9,12 @@ func IntSliceContains(sl []int, v int) bool {
 	}
 	return false
 }
+
+// ToSlice creates a slice with all string keys from a map
+func ToSlice(m map[string]struct{}) (s []string) {
+	for k := range m {
+		s = append(s, k)
+	}
+
+	return
+}
diff --git a/go.mod b/go.mod
index 9b1aeb0..9e8722b 100644
--- a/go.mod
+++ b/go.mod
@@ -14,6 +14,6 @@ require (
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.2.1
-	golang.org/x/net v0.0.0-20200822124328-c89045814202
+	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/text v0.3.3
 )
diff --git a/go.sum b/go.sum
index 11c0ad2..f239958 100644
--- a/go.sum
+++ b/go.sum
@@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
 github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
-github.com/projectdiscovery/rawhttp v0.0.0-20200901221031-6d215c865b2f h1:W6m8ZQL6nd3xzk/V7e1ScDNtff2iUkjKiv8RYyJlYlw=
-github.com/projectdiscovery/rawhttp v0.0.0-20200901221031-6d215c865b2f/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
 github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693 h1:dBBB/UXLVvy/Onb3YxVyg5ApwY8lRCIXt0owUosvNuI=
 github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
@@ -46,8 +44,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=