Merge branch 'dev' into feature-max-response-size

common/httputilz/httputilz.go

@@ -27,7 +27,7 @@ func DumpRequest(req *retryablehttp.Request) (string, error) {
 // DumpResponseHeadersAndRaw returns http headers and response as strings
 func DumpResponseHeadersAndRaw(resp *http.Response) (header, response string, err error) {
 	// httputil.DumpResponse does not work with websockets
-	if resp.StatusCode == http.StatusContinue {
+	if resp.StatusCode >= http.StatusContinue || resp.StatusCode <= http.StatusEarlyHints {
 		raw := resp.Status + "\n"
 		for h, v := range resp.Header {
 			raw += fmt.Sprintf("%s: %s\n", h, v)

internal/runner/options.go

@@ -51,6 +51,8 @@ type scanOptions struct {
 	NoFallback             bool
 	TechDetect             bool
 	MaxResponseBodySize    int
+	OutputExtractRegex     string
+	extractRegex           *regexp.Regexp
 }
 
 func (s *scanOptions) Clone() *scanOptions {
@@ -83,6 +85,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		PreferHTTPS:            s.PreferHTTPS,
 		NoFallback:             s.NoFallback,
 		TechDetect:             s.TechDetect,
+		OutputExtractRegex:     s.OutputExtractRegex,
 	}
 }
 
@@ -156,6 +159,7 @@ type Options struct {
 	ShowStatistics            bool
 	RandomAgent               bool
 	MaxResponseBodySize       int
+	OutputExtractRegex        string
 }
 
 // ParseOptions parses the command line options for application
@@ -220,6 +224,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)")
 	flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
 	flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size")
+	flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
 
 	flag.Parse()
 

internal/runner/runner.go

@@ -2,7 +2,6 @@ package runner
 
 import (
 	"bufio"
-	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -13,6 +12,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -172,6 +172,11 @@ func New(options *Options) (*Runner, error) {
 	scanopts.NoFallback = options.NoFallback
 	scanopts.TechDetect = options.TechDetect
 	scanopts.MaxResponseBodySize = options.MaxResponseBodySize
+	if options.OutputExtractRegex != "" {
+		if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
+			return nil, err
+		}
+	}
 
 	// output verb if more than one is specified
 	if len(scanopts.Methods) > 1 && !options.Silent {
@@ -547,9 +552,6 @@ retry:
 		req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody))
 	}
 
-	// Create a copy on the fly of the request body - ignore errors
-	bodyBytes, _ := req.BodyBytes()
-	req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 	requestDump, err := httputil.DumpRequestOut(req.Request, true)
 	if err != nil {
 		return Result{URL: URL, err: err}
@@ -751,6 +753,14 @@ retry:
 		}
 	}
 
+	// extract regex
+	if scanopts.extractRegex != nil {
+		matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1)
+		if len(matches) > 0 {
+			builder.WriteString(" [" + strings.Join(matches, ",") + "]")
+		}
+	}
+
 	// store responses in directory
 	if scanopts.StoreResponse {
 		domainFile := fmt.Sprintf("%s%s", domain, scanopts.RequestURI)