Merge branch 'dev' into added-whitelist-blacklist

.golangci.yml

@@ -73,13 +73,12 @@ linters:
     - gosimple
     - govet
     - ineffassign
-    - interfacer
     - misspell
     - nakedret
     - noctx
     - nolintlint
     - rowserrcheck
-    - scopelint
+    - exportloopref
     - staticcheck
     - structcheck
     - stylecheck

common/httpx/pipeline.go

@@ -55,7 +55,7 @@ func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool {
 	}
 
 	// expect at least 2 replies
-	return gotReplies >= 2
+	return gotReplies >= 2 //nolint
 }
 
 func pipelineDial(protocol, addr string) (net.Conn, error) {

internal/runner/options.go

@@ -2,6 +2,7 @@ package runner
 
 import (
 	"flag"
+	"math"
 	"os"
 	"regexp"
 
@@ -50,6 +51,7 @@ type scanOptions struct {
 	PreferHTTPS            bool
 	NoFallback             bool
 	TechDetect             bool
+	MaxResponseBodySize    int
 	OutputExtractRegex     string
 	extractRegex           *regexp.Regexp
 }
@@ -159,6 +161,7 @@ type Options struct {
 	RandomAgent               bool
 	Deny                      customlist.CustomList
 	Allow                     customlist.CustomList
+	MaxResponseBodySize       int
 	OutputExtractRegex        string
 }
 
@@ -225,6 +228,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
 	flag.Var(&options.Allow, "allow", "Whitelist ip/cidr")
 	flag.Var(&options.Deny, "deny", "Blacklist ip/cidr")
+	flag.IntVar(&options.MaxResponseBodySize, "max-response-body-size", math.MaxInt32, "Maximum response body size")
 	flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Extract Regex")
 
 	flag.Parse()

internal/runner/runner.go

@@ -173,6 +173,7 @@ func New(options *Options) (*Runner, error) {
 	scanopts.OutputResponseTime = options.OutputResponseTime
 	scanopts.NoFallback = options.NoFallback
 	scanopts.TechDetect = options.TechDetect
+	scanopts.MaxResponseBodySize = options.MaxResponseBodySize
 	if options.OutputExtractRegex != "" {
 		if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil {
 			return nil, err
@@ -375,7 +376,7 @@ func (r *Runner) RunEnumeration() {
 
 			row := resp.str
 			if r.options.JSONOutput {
-				row = resp.JSON()
+				row = resp.JSON(&r.scanopts)
 			}
 			gologger.Silent().Msgf("%s\n", row)
 			if f != nil {
@@ -777,7 +778,11 @@ retry:
 
 		domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt"
 		responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile)
-		writeErr := ioutil.WriteFile(responsePath, []byte(resp.Raw), 0644)
+		respRaw := resp.Raw
+		if len(respRaw) > scanopts.MaxResponseBodySize {
+			respRaw = respRaw[:scanopts.MaxResponseBodySize]
+		}
+		writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644)
 		if writeErr != nil {
 			gologger.Warning().Msgf("Could not write response, at path '%s', to disk: %s", responsePath, writeErr)
 		}
@@ -881,7 +886,11 @@ type Result struct {
 }
 
 // JSON the result
-func (r *Result) JSON() string {
+func (r Result) JSON(scanopts *scanOptions) string { //nolint
+	if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySize {
+		r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySize]
+	}
+
 	if js, err := json.Marshal(r); err == nil {
 		return string(js)
 	}