mirror of
https://github.com/projectdiscovery/httpx
synced 2026-06-08 16:50:17 +00:00
support multi -extract-regex flag and json output (#529)
* Add hashes flag, to support multi body hash type. issue:489 and issue:488 * go mod update * Add hash type validate for -hash flag * support multi extract regexps * support extract regexps matches for json output * support extract regexp preset * extract result deduplicate * small changes * lint ignore Co-authored-by: sandeep <sandeep@projectdiscovery.io> Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>
This commit is contained in:
M09Ic
@@ -0,0 +1,9 @@
|
||||
package customextract
|
||||
|
||||
import "regexp"
|
||||
|
||||
var ExtractPresets = map[string]*regexp.Regexp{
|
||||
"url": regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint
|
||||
"ip": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint
|
||||
"mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"), //nolint
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package customextract
|
||||
+5
-3
@@ -68,7 +68,7 @@ type scanOptions struct {
|
||||
MaxResponseBodySizeToSave int
|
||||
MaxResponseBodySizeToRead int
|
||||
OutputExtractRegex string
|
||||
extractRegex *regexp.Regexp
|
||||
extractRegexps map[string]*regexp.Regexp
|
||||
ExcludeCDN bool
|
||||
HostMaxErrors int
|
||||
ProbeAllIPS bool
|
||||
@@ -205,7 +205,8 @@ type Options struct {
|
||||
Allow customlist.CustomList
|
||||
MaxResponseBodySizeToSave int
|
||||
MaxResponseBodySizeToRead int
|
||||
OutputExtractRegex string
|
||||
OutputExtractRegexs goflags.StringSlice
|
||||
OutputExtractPresets goflags.StringSlice
|
||||
RateLimit int
|
||||
RateLimitMinute int
|
||||
Probe bool
|
||||
@@ -289,7 +290,8 @@ func ParseOptions() *Options {
|
||||
)
|
||||
|
||||
createGroup(flagSet, "extractor", "Extractor",
|
||||
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "display response content for specified regex"),
|
||||
flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex"),
|
||||
flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex"),
|
||||
)
|
||||
|
||||
createGroup(flagSet, "filters", "Filters",
|
||||
|
||||
+62
-36
@@ -22,6 +22,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/httpx/common/customextract"
|
||||
"github.com/projectdiscovery/sliceutil"
|
||||
|
||||
"github.com/ammario/ipisp/v2"
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/logrusorgru/aurora"
|
||||
@@ -212,9 +215,25 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.StoreChain = options.StoreChain
|
||||
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
|
||||
scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead
|
||||
if options.OutputExtractRegex != "" {
|
||||
if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
|
||||
return nil, err
|
||||
scanopts.extractRegexps = make(map[string]*regexp.Regexp)
|
||||
|
||||
if options.OutputExtractRegexs != nil {
|
||||
for _, regex := range options.OutputExtractRegexs {
|
||||
if compiledRegex, err := regexp.Compile(regex); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
scanopts.extractRegexps[regex] = compiledRegex
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if options.OutputExtractPresets != nil {
|
||||
for _, regexName := range options.OutputExtractPresets {
|
||||
if regex, ok := customextract.ExtractPresets[regexName]; ok {
|
||||
scanopts.extractRegexps[regexName] = regex
|
||||
} else {
|
||||
gologger.Warning().Msgf("Could not find preset: %s\n", regexName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1269,10 +1288,15 @@ retry:
|
||||
}
|
||||
|
||||
// extract regex
|
||||
if scanopts.extractRegex != nil {
|
||||
matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1)
|
||||
if len(matches) > 0 {
|
||||
builder.WriteString(" [" + strings.Join(matches, ",") + "]")
|
||||
var extractResult = map[string][]string{}
|
||||
if scanopts.extractRegexps != nil {
|
||||
for regex, compiledRegex := range scanopts.extractRegexps {
|
||||
matches := compiledRegex.FindAllString(string(resp.Data), -1)
|
||||
if len(matches) > 0 {
|
||||
matches = sliceutil.Dedupe(matches)
|
||||
builder.WriteString(" [" + strings.Join(matches, ",") + "]")
|
||||
extractResult[regex] = matches
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1475,6 +1499,7 @@ retry:
|
||||
FinalURL: finalURL,
|
||||
FavIconMMH3: faviconMMH3,
|
||||
Hashes: hashesMap,
|
||||
Extracts: extractResult,
|
||||
Jarm: jarmhash,
|
||||
Lines: resp.Lines,
|
||||
Words: resp.Words,
|
||||
@@ -1517,35 +1542,36 @@ type Result struct {
|
||||
Location string `json:"location,omitempty" csv:"location"`
|
||||
Title string `json:"title,omitempty" csv:"title"`
|
||||
str string
|
||||
Scheme string `json:"scheme,omitempty" csv:"scheme"`
|
||||
Error string `json:"error,omitempty" csv:"error"`
|
||||
WebServer string `json:"webserver,omitempty" csv:"webserver"`
|
||||
ResponseBody string `json:"response-body,omitempty" csv:"response-body"`
|
||||
ContentType string `json:"content-type,omitempty" csv:"content-type"`
|
||||
Method string `json:"method,omitempty" csv:"method"`
|
||||
Host string `json:"host,omitempty" csv:"host"`
|
||||
Path string `json:"path,omitempty" csv:"path"`
|
||||
FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
ResponseHeader string `json:"response-header,omitempty" csv:"response-header"`
|
||||
Request string `json:"request,omitempty" csv:"request"`
|
||||
ResponseTime string `json:"response-time,omitempty" csv:"response-time"`
|
||||
Jarm string `json:"jarm,omitempty" csv:"jarm"`
|
||||
ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"`
|
||||
A []string `json:"a,omitempty" csv:"a"`
|
||||
CNAMEs []string `json:"cnames,omitempty" csv:"cnames"`
|
||||
Technologies []string `json:"technologies,omitempty" csv:"technologies"`
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
StatusCode int `json:"status-code,omitempty" csv:"status-code"`
|
||||
ContentLength int `json:"content-length,omitempty" csv:"content-length"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
VHost bool `json:"vhost,omitempty" csv:"vhost"`
|
||||
WebSocket bool `json:"websocket,omitempty" csv:"websocket"`
|
||||
CDN bool `json:"cdn,omitempty" csv:"cdn"`
|
||||
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
|
||||
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
|
||||
Scheme string `json:"scheme,omitempty" csv:"scheme"`
|
||||
Error string `json:"error,omitempty" csv:"error"`
|
||||
WebServer string `json:"webserver,omitempty" csv:"webserver"`
|
||||
ResponseBody string `json:"response-body,omitempty" csv:"response-body"`
|
||||
ContentType string `json:"content-type,omitempty" csv:"content-type"`
|
||||
Method string `json:"method,omitempty" csv:"method"`
|
||||
Host string `json:"host,omitempty" csv:"host"`
|
||||
Path string `json:"path,omitempty" csv:"path"`
|
||||
FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"`
|
||||
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
|
||||
ResponseHeader string `json:"response-header,omitempty" csv:"response-header"`
|
||||
Request string `json:"request,omitempty" csv:"request"`
|
||||
ResponseTime string `json:"response-time,omitempty" csv:"response-time"`
|
||||
Jarm string `json:"jarm,omitempty" csv:"jarm"`
|
||||
ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"`
|
||||
A []string `json:"a,omitempty" csv:"a"`
|
||||
CNAMEs []string `json:"cnames,omitempty" csv:"cnames"`
|
||||
Technologies []string `json:"technologies,omitempty" csv:"technologies"`
|
||||
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"`
|
||||
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
|
||||
Words int `json:"words" csv:"words"`
|
||||
Lines int `json:"lines" csv:"lines"`
|
||||
StatusCode int `json:"status-code,omitempty" csv:"status-code"`
|
||||
ContentLength int `json:"content-length,omitempty" csv:"content-length"`
|
||||
Failed bool `json:"failed" csv:"failed"`
|
||||
VHost bool `json:"vhost,omitempty" csv:"vhost"`
|
||||
WebSocket bool `json:"websocket,omitempty" csv:"websocket"`
|
||||
CDN bool `json:"cdn,omitempty" csv:"cdn"`
|
||||
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
|
||||
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
|
||||
}
|
||||
|
||||
// JSON the result
|
||||
|
||||
Reference in New Issue
Block a user