Merge pull request #2462 from projectdiscovery/2418-store-response-only-matched

fixing output writing
2026-06-08 16:50:17 +00:00 · 2026-05-06 16:32:35 +03:00
parent a4a5b2e61f 98e6af00f0
commit 31af5c0be5
3 changed files with 221 additions and 42 deletions
@@ -343,7 +343,7 @@ type Options struct {
 	HeadlessOptionalArguments goflags.StringSlice
 	Protocol                  string
 	OutputFilterErrorPagePath string
-	DisableStdout             bool
+	DisableStdout bool

 	JavascriptCodes goflags.StringSlice

@@ -696,6 +696,30 @@ func ParseOptions() *Options {
 	return options
 }

+func (options *Options) HasMatcherOrFilter() bool {
+	return len(options.matchStatusCode) > 0 ||
+		len(options.matchContentLength) > 0 ||
+		len(options.filterStatusCode) > 0 ||
+		len(options.filterContentLength) > 0 ||
+		len(options.matchRegexes) > 0 ||
+		len(options.filterRegexes) > 0 ||
+		len(options.matchLinesCount) > 0 ||
+		len(options.matchWordsCount) > 0 ||
+		len(options.filterLinesCount) > 0 ||
+		len(options.filterWordsCount) > 0 ||
+		len(options.OutputMatchString) > 0 ||
+		len(options.OutputFilterString) > 0 ||
+		len(options.OutputMatchFavicon) > 0 ||
+		len(options.OutputFilterFavicon) > 0 ||
+		len(options.OutputMatchCdn) > 0 ||
+		len(options.OutputFilterCdn) > 0 ||
+		len(options.OutputFilterPageType) > 0 ||
+		options.OutputMatchCondition != "" ||
+		options.OutputFilterCondition != "" ||
+		options.OutputMatchResponseTime != "" ||
+		options.OutputFilterResponseTime != ""
+}
+
 func (options *Options) ValidateOptions() error {
 	if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) {
 		return fmt.Errorf("file '%s' does not exist", options.InputFile)
@@ -2458,53 +2458,56 @@ retry:
 	responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)

 	var responsePath, fileNameHash string
-	// store response
+	// store response — when matchers/filters are active, defer writing to the
+	// output loop so only matched responses are persisted to disk.
 	if scanopts.StoreResponse || scanopts.StoreChain {
-		if r.options.OmitBody {
-			resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "")
-		}
-		responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
-		// URL.EscapedString returns that can be used as filename
-		respRaw := resp.Raw
-		reqRaw := requestDump
-		if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
-			respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
-		}
-		data := reqRaw
-		if scanopts.StoreChain && resp.HasChain() {
-			data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...)
-		}
-		data = append(data, respRaw...)
-		data = append(data, []byte("\n\n\n")...)
-		data = append(data, []byte(fullURL)...)
-		_ = fileutil.CreateFolder(responseBaseDir)
+		fileNameHash = hash

-		basePath := strings.TrimSuffix(responsePath, ".txt")
-		var idx int
-		for idx = 0; ; idx++ {
-			targetPath := responsePath
-			if idx > 0 {
-				targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
+		if !r.options.HasMatcherOrFilter() {
+			if r.options.OmitBody {
+				resp.Raw = strings.ReplaceAll(resp.Raw, string(resp.Data), "")
 			}
-			f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
-			if err == nil {
-				_, writeErr := f.Write(data)
-				_ = f.Close()
-				if writeErr != nil {
-					gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
+			responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
+			// URL.EscapedString returns that can be used as filename
+			respRaw := resp.Raw
+			reqRaw := requestDump
+			if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
+				respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
+			}
+			data := reqRaw
+			if scanopts.StoreChain && resp.HasChain() {
+				data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...)
+			}
+			data = append(data, respRaw...)
+			data = append(data, []byte("\n\n\n")...)
+			data = append(data, []byte(fullURL)...)
+			_ = fileutil.CreateFolder(responseBaseDir)
+
+			basePath := strings.TrimSuffix(responsePath, ".txt")
+			var idx int
+			for idx = 0; ; idx++ {
+				targetPath := responsePath
+				if idx > 0 {
+					targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
+				}
+				f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
+				if err == nil {
+					_, writeErr := f.Write(data)
+					_ = f.Close()
+					if writeErr != nil {
+						gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
+					}
+					break
+				}
+				if !os.IsExist(err) {
+					gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
+					break
 				}
-				break
 			}
-			if !os.IsExist(err) {
-				gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
-				break
-			}
-		}

-		if idx == 0 {
-			fileNameHash = hash
-		} else {
-			fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
+			if idx > 0 {
+				fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
+			}
 		}
 	}

@@ -386,6 +386,158 @@ func TestRunner_testAndSet_concurrent(t *testing.T) {
 	require.Equal(t, 1, winCount, "exactly one goroutine should win testAndSet for the same key")
 }

+func TestOptions_hasMatcherOrFilter(t *testing.T) {
+	tests := []struct {
+		name     string
+		options  Options
+		expected bool
+	}{
+		{
+			name:     "no matchers or filters",
+			options:  Options{},
+			expected: false,
+		},
+		{
+			name:     "match status code",
+			options:  Options{OutputMatchStatusCode: "200"},
+			expected: true,
+		},
+		{
+			name:     "filter status code",
+			options:  Options{OutputFilterStatusCode: "403,401"},
+			expected: true,
+		},
+		{
+			name:     "match string",
+			options:  Options{OutputMatchString: []string{"admin"}},
+			expected: true,
+		},
+		{
+			name:     "filter string",
+			options:  Options{OutputFilterString: []string{"error"}},
+			expected: true,
+		},
+		{
+			name:     "match content length",
+			options:  Options{OutputMatchContentLength: "100"},
+			expected: true,
+		},
+		{
+			name:     "filter content length",
+			options:  Options{OutputFilterContentLength: "0"},
+			expected: true,
+		},
+		{
+			name:     "match regex",
+			options:  Options{OutputMatchRegex: []string{"admin.*panel"}},
+			expected: true,
+		},
+		{
+			name:     "filter regex",
+			options:  Options{OutputFilterRegex: []string{"error"}},
+			expected: true,
+		},
+		{
+			name:     "match lines count",
+			options:  Options{OutputMatchLinesCount: "50"},
+			expected: true,
+		},
+		{
+			name:     "filter lines count",
+			options:  Options{OutputFilterLinesCount: "0"},
+			expected: true,
+		},
+		{
+			name:     "match words count",
+			options:  Options{OutputMatchWordsCount: "100"},
+			expected: true,
+		},
+		{
+			name:     "filter words count",
+			options:  Options{OutputFilterWordsCount: "0"},
+			expected: true,
+		},
+		{
+			name:     "match favicon",
+			options:  Options{OutputMatchFavicon: []string{"1494302000"}},
+			expected: true,
+		},
+		{
+			name:     "filter favicon",
+			options:  Options{OutputFilterFavicon: []string{"1494302000"}},
+			expected: true,
+		},
+		{
+			name:     "match cdn",
+			options:  Options{OutputMatchCdn: []string{"cloudflare"}},
+			expected: true,
+		},
+		{
+			name:     "filter cdn",
+			options:  Options{OutputFilterCdn: []string{"cloudflare"}},
+			expected: true,
+		},
+		{
+			name:     "match condition",
+			options:  Options{OutputMatchCondition: "status_code == 200"},
+			expected: true,
+		},
+		{
+			name:     "filter condition",
+			options:  Options{OutputFilterCondition: "status_code == 403"},
+			expected: true,
+		},
+		{
+			name:     "match response time",
+			options:  Options{OutputMatchResponseTime: "< 1"},
+			expected: true,
+		},
+		{
+			name:     "filter response time",
+			options:  Options{OutputFilterResponseTime: "> 5"},
+			expected: true,
+		},
+		{
+			name:     "filter page type",
+			options:  Options{OutputFilterPageType: []string{"error"}},
+			expected: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			opts := tc.options
+			err := opts.ValidateOptions()
+			require.Nil(t, err)
+			require.Equal(t, tc.expected, opts.HasMatcherOrFilter(),
+				"HasMatcherOrFilter() should be %v for %s", tc.expected, tc.name)
+		})
+	}
+}
+
+func TestStoreResponse_withoutMatchersStoresAll(t *testing.T) {
+	dir := t.TempDir()
+	opts := &Options{
+		StoreResponse:    true,
+		StoreResponseDir: dir,
+	}
+	err := opts.ValidateOptions()
+	require.Nil(t, err)
+	require.False(t, opts.HasMatcherOrFilter())
+}
+
+func TestStoreResponse_withMatcherSetsFlag(t *testing.T) {
+	dir := t.TempDir()
+	opts := &Options{
+		StoreResponse:       true,
+		StoreResponseDir:    dir,
+		OutputMatchStatusCode: "200",
+	}
+	err := opts.ValidateOptions()
+	require.Nil(t, err)
+	require.True(t, opts.HasMatcherOrFilter())
+}
+
 func TestRunner_duplicate(t *testing.T) {
 	const (
 		pageA = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<html><head><title>Welcome</title></head><body>Hello world default page content here</body></html>"