admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding trace (#1883)

Browse Source 
* adding trace

* using retryablehttp

* syntax error

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Mzack9999
2024-09-16 20:26:21 +02:00
committed by GitHub
parent d6082fd505
commit db72bb2902
5 changed files with 101 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/httpx/httpx.go
+1 -1
View File
@@ -76,7 +76,7 @@ func New(options *Options) (*HTTPX, error) {
var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying
retryablehttpOptions.Timeout = httpx.Options.Timeout
retryablehttpOptions.RetryMax = httpx.Options.RetryMax
retryablehttpOptions.Trace = options.Trace
handleHSTS := func(req *http.Request) {
if req.Response.Header.Get("Strict-Transport-Security") == "" {
return
common/httpx/option.go
+1
View File
@@ -49,6 +49,7 @@ type Options struct {
NetworkPolicy *networkpolicy.NetworkPolicy
CDNCheckClient *cdncheck.Client
Protocol Proto
Trace bool
}
// DefaultOptions contains the default options
runner/options.go
+3
View File
@@ -329,6 +329,8 @@ type Options struct {
// to be exact at end of existing closures
OnClose func()
Trace bool
// Optional pre-created objects to reduce allocations
Wappalyzer *wappalyzer.Wappalyze
Networkpolicy *networkpolicy.NetworkPolicy
@@ -500,6 +502,7 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"),
flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"),
flagSet.BoolVarP(&options.Trace, "trace", "tr", false, "trace"),
)
flagSet.CreateGroup("Optimizations", "Optimizations",
runner/runner.go
+11 -4
View File
@@ -126,6 +126,7 @@ func New(options *Options) (*Runner, error) {
}
httpxOptions := httpx.DefaultOptions
httpxOptions.Trace = options.Trace
var np *networkpolicy.NetworkPolicy
if options.Networkpolicy != nil {
@@ -1523,7 +1524,10 @@ retry:
if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil {
gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI)
}
var req *retryablehttp.Request
var (
req *retryablehttp.Request
ctx context.Context
)
if target.CustomIP != "" {
var requestIP string
if iputil.IsIPv6(target.CustomIP) {
@@ -1531,11 +1535,11 @@ retry:
} else {
requestIP = target.CustomIP
}
ctx := context.WithValue(context.Background(), fastdialer.IP, requestIP)
req, err = hp.NewRequestWithContext(ctx, method, URL.String())
ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP)
} else {
req, err = hp.NewRequest(method, URL.String())
ctx = context.Background()
}
req, err = hp.NewRequestWithContext(ctx, method, URL.String())
if err != nil {
return Result{URL: URL.String(), Input: origInput, Err: err}
}
@@ -2252,6 +2256,9 @@ retry:
result.Fqdns = resp.BodyDomains.Fqdns
result.Domains = resp.BodyDomains.Domains
}
if r.options.Trace {
result.Trace = req.TraceInfo
}
return result
}
runner/types.go
+85 -67
View File
@@ -12,6 +12,7 @@ import (
mapstructure "github.com/go-viper/mapstructure/v2"
"github.com/projectdiscovery/dsl"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
mapsutil "github.com/projectdiscovery/utils/maps"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
@@ -32,74 +33,91 @@ func (o AsnResponse) String() string {
// Result of a scan
type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
Err error `json:"-" csv:"-" mapstructure:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
Port string `json:"port,omitempty" csv:"port" mapstructure:"port"`
Raw string `json:"-" csv:"-" mapstructure:"-"`
URL string `json:"url,omitempty" csv:"url" mapstructure:"url"`
Input string `json:"input,omitempty" csv:"input" mapstructure:"input"`
Location string `json:"location,omitempty" csv:"location" mapstructure:"location"`
Title string `json:"title,omitempty" csv:"title" mapstructure:"title"`
str string `mapstructure:"-"`
Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
Error string `json:"error,omitempty" csv:"error" mapstructure:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
Method string `json:"method,omitempty" csv:"method" mapstructure:"method"`
Host string `json:"host,omitempty" csv:"host" mapstructure:"host"`
Path string `json:"path,omitempty" csv:"path" mapstructure:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"`
Request string `json:"request,omitempty" csv:"request" mapstructure:"request"`
ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a" mapstructure:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
Words int `json:"words" csv:"words" mapstructure:"words"`
Lines int `json:"lines" csv:"lines" mapstructure:"lines"`
StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"`
ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"`
Failed bool `json:"failed" csv:"failed" mapstructure:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
Domains []string `json:"body_domains,omitempty" mapstructure:"body_domains"`
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
Err error `json:"-" csv:"-" mapstructure:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
Port string `json:"port,omitempty" csv:"port" mapstructure:"port"`
Raw string `json:"-" csv:"-" mapstructure:"-"`
URL string `json:"url,omitempty" csv:"url" mapstructure:"url"`
Input string `json:"input,omitempty" csv:"input" mapstructure:"input"`
Location string `json:"location,omitempty" csv:"location" mapstructure:"location"`
Title string `json:"title,omitempty" csv:"title" mapstructure:"title"`
str string `mapstructure:"-"`
Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
Error string `json:"error,omitempty" csv:"error" mapstructure:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
Method string `json:"method,omitempty" csv:"method" mapstructure:"method"`
Host string `json:"host,omitempty" csv:"host" mapstructure:"host"`
Path string `json:"path,omitempty" csv:"path" mapstructure:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"`
Request string `json:"request,omitempty" csv:"request" mapstructure:"request"`
ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a" mapstructure:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
Words int `json:"words" csv:"words" mapstructure:"words"`
Lines int `json:"lines" csv:"lines" mapstructure:"lines"`
StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"`
ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"`
Failed bool `json:"failed" csv:"failed" mapstructure:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
Domains []string `json:"body_domains,omitempty" mapstructure:"body_domains"`
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"trace" mapstructure:"trace"`
}
// Internal Fields
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
type Trace struct {
GetConn time.Time `json:"get_conn,omitempty"`
GotConn time.Time `json:"got_conn,omitempty"`
PutIdleConn time.Time `json:"put_idle_conn,omitempty"`
GotFirstResponseByte time.Time `json:"got_first_response_byte,omitempty"`
Got100Continue time.Time `json:"got_100_continue,omitempty"`
DNSStart time.Time `json:"dns_start,omitempty"`
DNSDone time.Time `json:"dns_done,omitempty"`
ConnectStart time.Time `json:"connect_start,omitempty"`
ConnectDone time.Time `json:"connect_done,omitempty"`
TLSHandshakeStart time.Time `json:"tls_handshake_start,omitempty"`
TLSHandshakeDone time.Time `json:"tls_handshake_done,omitempty"`
WroteHeaderField time.Time `json:"wrote_header_field,omitempty"`
WroteHeaders time.Time `json:"wrote_headers,omitempty"`
Wait100Continue time.Time `json:"wait_100_continue,omitempty"`
WroteRequest time.Time `json:"wrote_request,omitempty"`
}
// function to get dsl variables from result struct