admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #104 from projectdiscovery/bugfix-csp-multiple-headers

Browse Source 
extended CSP extraction
This commit is contained in:
bauthard
2020-09-14 14:27:29 +05:30
committed by GitHub
parent 5f2f9c3bd6 ca20e5538c
commit 6abe83e2f8
4 changed files with 39 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/httpx/csp.go
+27 -13
View File
@@ -3,33 +3,47 @@ package httpx
import (
"net/http"
"strings"
"github.com/projectdiscovery/httpx/common/slice"
)
// CSPHeaders is an incomplete list of most common CSP headers
var CSPHeaders []string = []string{
"Content-Security-Policy", //standard
"Content-Security-Policy-Report-Only", //standard
"X-Content-Security-Policy-Report-Only", // non - standard
"X-Webkit-Csp-Report-Only", // non - standard
}
type CspData struct {
Domains []string `json:"domains,omitempty"`
}
func (h *HTTPX) CspGrab(r *http.Response) *CspData {
cspRaw := r.Header.Get("Content-Security-Policy")
if cspRaw != "" {
var domains []string
rules := strings.Split(cspRaw, ";")
for _, rule := range rules {
// rule is like aa bb domain1 domain2 domain3
tokens := strings.Split(rule, " ")
// we extracts only potential domains
for _, t := range tokens {
if isPotentialDomain(t) {
domains = append(domains, t)
domains := make(map[string]struct{})
for _, cspHeader := range CSPHeaders {
cspRaw := r.Header.Get(cspHeader)
if cspRaw != "" {
rules := strings.Split(cspRaw, ";")
for _, rule := range rules {
// rule is like aa bb domain1 domain2 domain3
tokens := strings.Split(rule, " ")
// we extracts only potential domains
for _, t := range tokens {
if isPotentialDomain(t) {
domains[t] = struct{}{}
}
}
}
}
return &CspData{Domains: domains}
}
if len(domains) > 0 {
return &CspData{Domains: slice.ToSlice(domains)}
}
return nil
}
// bare minimum conditions to filter potential domains
func isPotentialDomain(s string) bool {
return strings.Contains(s, ".") || strings.HasPrefix(s, "http")
}
common/slice/slice.go
+9
View File
@@ -9,3 +9,12 @@ func IntSliceContains(sl []int, v int) bool {
}
return false
}
// ToSlice creates a slice with all string keys from a map
func ToSlice(m map[string]struct{}) (s []string) {
for k := range m {
s = append(s, k)
}
return
}
go.mod
+1 -1
View File
@@ -14,6 +14,6 @@ require (
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.2.1
golang.org/x/net v0.0.0-20200822124328-c89045814202
golang.org/x/net v0.0.0-20200904194848-62affa334b73
golang.org/x/text v0.3.3
)
go.sum
+2 -4
View File
@@ -24,8 +24,6 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
github.com/projectdiscovery/rawhttp v0.0.0-20200901221031-6d215c865b2f h1:W6m8ZQL6nd3xzk/V7e1ScDNtff2iUkjKiv8RYyJlYlw=
github.com/projectdiscovery/rawhttp v0.0.0-20200901221031-6d215c865b2f/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693 h1:dBBB/UXLVvy/Onb3YxVyg5ApwY8lRCIXt0owUosvNuI=
github.com/projectdiscovery/rawhttp v0.0.0-20200901223513-8a8a0cef2693/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
@@ -46,8 +44,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=