diff --git a/common/customextract/customextract.go b/common/customextract/customextract.go new file mode 100644 index 0000000..d15afca --- /dev/null +++ b/common/customextract/customextract.go @@ -0,0 +1,9 @@ +package customextract + +import "regexp" + +var ExtractPresets = map[string]*regexp.Regexp{ + "url": regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint + "ip": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint + "mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"), //nolint +} diff --git a/common/customextract/doc.go b/common/customextract/doc.go new file mode 100644 index 0000000..20f24a8 --- /dev/null +++ b/common/customextract/doc.go @@ -0,0 +1 @@ +package customextract diff --git a/runner/options.go b/runner/options.go index 25407e3..43132b2 100644 --- a/runner/options.go +++ b/runner/options.go @@ -68,7 +68,7 @@ type scanOptions struct { MaxResponseBodySizeToSave int MaxResponseBodySizeToRead int OutputExtractRegex string - extractRegex *regexp.Regexp + extractRegexps map[string]*regexp.Regexp ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool @@ -205,7 +205,8 @@ type Options struct { Allow customlist.CustomList MaxResponseBodySizeToSave int MaxResponseBodySizeToRead int - OutputExtractRegex string + OutputExtractRegexs goflags.StringSlice + OutputExtractPresets goflags.StringSlice RateLimit int RateLimitMinute int Probe bool @@ -289,7 +290,8 @@ func ParseOptions() *Options { ) createGroup(flagSet, "extractor", "Extractor", - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "display response content for specified regex"), + flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex"), + flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex"), ) createGroup(flagSet, "filters", "Filters", diff --git a/runner/runner.go b/runner/runner.go index 610d2f1..d046aa9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -22,6 +22,9 @@ import ( "strings" "time" + "github.com/projectdiscovery/httpx/common/customextract" + "github.com/projectdiscovery/sliceutil" + "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" "github.com/logrusorgru/aurora" @@ -212,9 +215,25 @@ func New(options *Options) (*Runner, error) { scanopts.StoreChain = options.StoreChain scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead - if options.OutputExtractRegex != "" { - if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil { - return nil, err + scanopts.extractRegexps = make(map[string]*regexp.Regexp) + + if options.OutputExtractRegexs != nil { + for _, regex := range options.OutputExtractRegexs { + if compiledRegex, err := regexp.Compile(regex); err != nil { + return nil, err + } else { + scanopts.extractRegexps[regex] = compiledRegex + } + } + } + + if options.OutputExtractPresets != nil { + for _, regexName := range options.OutputExtractPresets { + if regex, ok := customextract.ExtractPresets[regexName]; ok { + scanopts.extractRegexps[regexName] = regex + } else { + gologger.Warning().Msgf("Could not find preset: %s\n", regexName) + } } } @@ -1269,10 +1288,15 @@ retry: } // extract regex - if scanopts.extractRegex != nil { - matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1) - if len(matches) > 0 { - builder.WriteString(" [" + strings.Join(matches, ",") + "]") + var extractResult = map[string][]string{} + if scanopts.extractRegexps != nil { + for regex, compiledRegex := range scanopts.extractRegexps { + matches := compiledRegex.FindAllString(string(resp.Data), -1) + if len(matches) > 0 { + matches = sliceutil.Dedupe(matches) + builder.WriteString(" [" + strings.Join(matches, ",") + "]") + extractResult[regex] = matches + } } } @@ -1475,6 +1499,7 @@ retry: FinalURL: finalURL, FavIconMMH3: faviconMMH3, Hashes: hashesMap, + Extracts: extractResult, Jarm: jarmhash, Lines: resp.Lines, Words: resp.Words, @@ -1517,35 +1542,36 @@ type Result struct { Location string `json:"location,omitempty" csv:"location"` Title string `json:"title,omitempty" csv:"title"` str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"response-body,omitempty" csv:"response-body"` - ContentType string `json:"content-type,omitempty" csv:"content-type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` - FinalURL string `json:"final-url,omitempty" csv:"final-url"` - ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"response-time,omitempty" csv:"response-time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` - Technologies []string `json:"technologies,omitempty" csv:"technologies"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status-code,omitempty" csv:"status-code"` - ContentLength int `json:"content-length,omitempty" csv:"content-length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"response-body,omitempty" csv:"response-body"` + ContentType string `json:"content-type,omitempty" csv:"content-type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + FinalURL string `json:"final-url,omitempty" csv:"final-url"` + ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Technologies []string `json:"technologies,omitempty" csv:"technologies"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status-code,omitempty" csv:"status-code"` + ContentLength int `json:"content-length,omitempty" csv:"content-length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` } // JSON the result