mirror of
https://github.com/projectdiscovery/httpx
synced 2026-06-08 16:50:17 +00:00
ayuxsec@proton.me
193 lines
12 KiB
Go
193 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-faker/faker/v4"
|
|
"github.com/go-faker/faker/v4/pkg/options"
|
|
mapstructure "github.com/go-viper/mapstructure/v2"
|
|
"github.com/projectdiscovery/dsl"
|
|
"github.com/projectdiscovery/gologger"
|
|
"github.com/projectdiscovery/retryablehttp-go"
|
|
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
|
|
mapsutil "github.com/projectdiscovery/utils/maps"
|
|
wappalyzer "github.com/projectdiscovery/wappalyzergo"
|
|
|
|
"github.com/projectdiscovery/httpx/common/httpx"
|
|
)
|
|
|
|
type AsnResponse struct {
|
|
AsNumber string `json:"as_number" csv:"as_number"`
|
|
AsName string `json:"as_name" csv:"as_name"`
|
|
AsCountry string `json:"as_country" csv:"as_country"`
|
|
AsRange []string `json:"as_range" csv:"as_range"`
|
|
}
|
|
|
|
func (o AsnResponse) String() string {
|
|
return fmt.Sprintf("%v, %v, %v", o.AsNumber, o.AsName, o.AsCountry)
|
|
}
|
|
|
|
// Result of a scan
|
|
type Result struct {
|
|
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" md:"timestamp" mapstructure:"timestamp"`
|
|
LinkRequest []NetworkRequest `json:"link_request,omitempty" csv:"link_request" md:"link_request" mapstructure:"link_request"`
|
|
ASN *AsnResponse `json:"asn,omitempty" csv:"-" md:"-" mapstructure:"asn"`
|
|
Err error `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"-" md:"-" mapstructure:"csp"`
|
|
TLSData *clients.Response `json:"tls,omitempty" csv:"-" md:"-" mapstructure:"tls"`
|
|
Hashes map[string]interface{} `json:"hash,omitempty" csv:"-" md:"-" mapstructure:"hash"`
|
|
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" md:"extract_regex" mapstructure:"extract_regex"`
|
|
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" md:"cdn_name" mapstructure:"cdn_name"`
|
|
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" md:"cdn_type" mapstructure:"cdn_type"`
|
|
SNI string `json:"sni,omitempty" csv:"sni" md:"sni" mapstructure:"sni"`
|
|
Port string `json:"port,omitempty" csv:"port" md:"port" mapstructure:"port"`
|
|
Raw string `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
URL string `json:"url,omitempty" csv:"url" md:"url" mapstructure:"url"`
|
|
Input string `json:"input,omitempty" csv:"input" md:"input" mapstructure:"input"`
|
|
Location string `json:"location,omitempty" csv:"location" md:"location" mapstructure:"location"`
|
|
Title string `json:"title,omitempty" csv:"title" md:"title" mapstructure:"title"`
|
|
str string `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
Scheme string `json:"scheme,omitempty" csv:"scheme" md:"scheme" mapstructure:"scheme"`
|
|
Error string `json:"error,omitempty" csv:"error" md:"error" mapstructure:"error"`
|
|
WebServer string `json:"webserver,omitempty" csv:"webserver" md:"webserver" mapstructure:"webserver"`
|
|
ResponseBody string `json:"body,omitempty" csv:"-" md:"-" mapstructure:"body"`
|
|
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" md:"body_preview" mapstructure:"body_preview"`
|
|
ContentType string `json:"content_type,omitempty" csv:"content_type" md:"content_type" mapstructure:"content_type"`
|
|
Method string `json:"method,omitempty" csv:"method" md:"method" mapstructure:"method"`
|
|
Host string `json:"host,omitempty" csv:"host" md:"host" mapstructure:"host"`
|
|
HostIP string `json:"host_ip,omitempty" csv:"host_ip" md:"host_ip" mapstructure:"host_ip"`
|
|
Path string `json:"path,omitempty" csv:"path" md:"path" mapstructure:"path"`
|
|
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" md:"favicon" mapstructure:"favicon"`
|
|
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" md:"favicon_md5" mapstructure:"favicon_md5"`
|
|
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" md:"favicon_path" mapstructure:"favicon_path"`
|
|
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" md:"favicon_url" mapstructure:"favicon_url"`
|
|
FinalURL string `json:"final_url,omitempty" csv:"final_url" md:"final_url" mapstructure:"final_url"`
|
|
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" md:"-" mapstructure:"header"`
|
|
RawHeaders string `json:"raw_header,omitempty" csv:"-" md:"-" mapstructure:"raw_header"`
|
|
Request string `json:"request,omitempty" csv:"-" md:"-" mapstructure:"request"`
|
|
ResponseTime string `json:"time,omitempty" csv:"time" md:"time" mapstructure:"time"`
|
|
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" md:"jarm_hash" mapstructure:"jarm_hash"`
|
|
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" md:"chain_status_codes" mapstructure:"chain_status_codes"`
|
|
A []string `json:"a,omitempty" csv:"a" md:"a" mapstructure:"a"`
|
|
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" md:"aaaa" mapstructure:"aaaa"`
|
|
CNAMEs []string `json:"cname,omitempty" csv:"cname" md:"cname" mapstructure:"cname"`
|
|
Technologies []string `json:"tech,omitempty" csv:"tech" md:"tech" mapstructure:"tech"`
|
|
Extracts map[string][]string `json:"extracts,omitempty" csv:"-" md:"-" mapstructure:"extracts"`
|
|
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"-" md:"-" mapstructure:"chain"`
|
|
Words int `json:"words" csv:"words" md:"words" mapstructure:"words"`
|
|
Lines int `json:"lines" csv:"lines" md:"lines" mapstructure:"lines"`
|
|
StatusCode int `json:"status_code" csv:"status_code" md:"status_code" mapstructure:"status_code"`
|
|
ContentLength int `json:"content_length" csv:"content_length" md:"content_length" mapstructure:"content_length"`
|
|
Failed bool `json:"failed" csv:"failed" md:"failed" mapstructure:"failed"`
|
|
VHost bool `json:"vhost,omitempty" csv:"vhost" md:"vhost" mapstructure:"vhost"`
|
|
WebSocket bool `json:"websocket,omitempty" csv:"websocket" md:"websocket" mapstructure:"websocket"`
|
|
CDN bool `json:"cdn,omitempty" csv:"cdn" md:"cdn" mapstructure:"cdn"`
|
|
HTTP2 bool `json:"http2,omitempty" csv:"http2" md:"http2" mapstructure:"http2"`
|
|
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" md:"pipeline" mapstructure:"pipeline"`
|
|
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" md:"headless_body" mapstructure:"headless_body"`
|
|
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" md:"screenshot_bytes" mapstructure:"screenshot_bytes"`
|
|
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" md:"stored_response_path" mapstructure:"stored_response_path"`
|
|
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" md:"screenshot_path" mapstructure:"screenshot_path"`
|
|
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" md:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
|
|
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"-" md:"-" mapstructure:"knowledgebase"`
|
|
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" md:"resolvers" mapstructure:"resolvers"`
|
|
Fqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn" md:"body_fqdn" mapstructure:"body_fqdn"`
|
|
Domains []string `json:"body_domains,omitempty" csv:"body_domains" md:"body_domains" mapstructure:"body_domains"`
|
|
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
RequestRaw []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"`
|
|
FileNameHash string `json:"-" csv:"-" md:"-" mapstructure:"-"`
|
|
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" md:"cpe" mapstructure:"cpe"`
|
|
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" md:"wordpress" mapstructure:"wordpress"`
|
|
}
|
|
|
|
type Trace struct {
|
|
GetConn time.Time `json:"get_conn,omitempty"`
|
|
GotConn time.Time `json:"got_conn,omitempty"`
|
|
PutIdleConn time.Time `json:"put_idle_conn,omitempty"`
|
|
GotFirstResponseByte time.Time `json:"got_first_response_byte,omitempty"`
|
|
Got100Continue time.Time `json:"got_100_continue,omitempty"`
|
|
DNSStart time.Time `json:"dns_start,omitempty"`
|
|
DNSDone time.Time `json:"dns_done,omitempty"`
|
|
ConnectStart time.Time `json:"connect_start,omitempty"`
|
|
ConnectDone time.Time `json:"connect_done,omitempty"`
|
|
TLSHandshakeStart time.Time `json:"tls_handshake_start,omitempty"`
|
|
TLSHandshakeDone time.Time `json:"tls_handshake_done,omitempty"`
|
|
WroteHeaderField time.Time `json:"wrote_header_field,omitempty"`
|
|
WroteHeaders time.Time `json:"wrote_headers,omitempty"`
|
|
Wait100Continue time.Time `json:"wait_100_continue,omitempty"`
|
|
WroteRequest time.Time `json:"wrote_request,omitempty"`
|
|
}
|
|
|
|
// function to get dsl variables from result struct
|
|
func dslVariables() ([]string, error) {
|
|
fakeResult := Result{}
|
|
fieldsToIgnore := []string{"Hashes", "ResponseHeaders", "Err", "KnowledgeBase"}
|
|
if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...), options.WithIgnoreInterface(true)); err != nil {
|
|
return nil, err
|
|
}
|
|
m, err := resultToMap(fakeResult)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
vars := []string{"header_md5", "header_mmh3", "header_sha256", "header_simhash", "body_md5", "body_mmh3", "body_sha256", "body_simhash"}
|
|
mapsutil.Walk(m, func(k string, v any) {
|
|
vars = append(vars, k)
|
|
})
|
|
|
|
return vars, nil
|
|
}
|
|
|
|
func evalDslExpr(result Result, dslExpr string) bool {
|
|
resultMap, err := resultToMap(result)
|
|
if err != nil {
|
|
gologger.Warning().Msgf("Could not map result: %s\n", err)
|
|
return false
|
|
}
|
|
|
|
res, err := dsl.EvalExpr(dslExpr, resultMap)
|
|
if err != nil && !ignoreErr(err) {
|
|
gologger.Error().Msgf("Could not evaluate DSL expression: %s\n", err)
|
|
return false
|
|
}
|
|
return res == true
|
|
}
|
|
|
|
func resultToMap(resp Result) (map[string]any, error) {
|
|
m := make(map[string]any)
|
|
config := &mapstructure.DecoderConfig{
|
|
Result: &m,
|
|
}
|
|
decoder, err := mapstructure.NewDecoder(config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating decoder: %v", err)
|
|
}
|
|
err = decoder.Decode(resp)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error decoding: %v", err)
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
var (
|
|
// showDSLErr controls whether to show hidden DSL errors or not
|
|
showDSLErr = strings.EqualFold(os.Getenv("SHOW_DSL_ERRORS"), "true")
|
|
)
|
|
|
|
// ignoreErr checks if the error is to be ignored or not
|
|
func ignoreErr(err error) bool {
|
|
if showDSLErr {
|
|
return false
|
|
}
|
|
if errors.Is(err, dsl.ErrParsingArg) || strings.Contains(err.Error(), "No parameter") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|