diff --git a/.gitignore b/.gitignore index 028e9b1..7ff53c9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg + +.devcontainer \ No newline at end of file diff --git a/README.md b/README.md index 4355b1c..3ce6f76 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ UPDATE: OUTPUT: -o, -output string file to write output results + -oa, -output-all filename to write output results in all formats -sr, -store-response store http response to output directory -srd, -store-response-dir string store http response to custom directory -csv store output in csv format diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 654d50c..62bc34e 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -5,10 +5,12 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "github.com/julienschmidt/httprouter" "github.com/projectdiscovery/httpx/internal/testutils" + fileutil "github.com/projectdiscovery/utils/file" ) var httpTestcases = map[string]testutils.TestCase{ @@ -30,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}}, "Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}}, "Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}}, + "Output All": &outputAll{}, } type standardHttpGet struct { @@ -377,3 +380,42 @@ func (h *outputFilterCondition) Execute() error { } return nil } + +type outputAll struct { +} + +func (h *outputAll) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + fmt.Fprint(w, `{"status": "ok"}`) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + fileName := "test_output_all" + _, hErr := testutils.RunHttpxAndGetResults(ts.URL, false, []string{"-o", fileName, "-oa"}...) + if hErr != nil { + return hErr + } + + expectedFiles := []string{fileName, fileName + ".json", fileName + ".csv"} + var actualFiles []string + + for _, file := range expectedFiles { + if fileutil.FileExists(file) { + actualFiles = append(actualFiles, file) + } + } + if len(actualFiles) != 3 { + return errIncorrectResultsCount(actualFiles) + } + + for _, file := range actualFiles { + _ = os.Remove(file) + } + + return nil +} diff --git a/runner/options.go b/runner/options.go index 2cd90c8..bbf0ef6 100644 --- a/runner/options.go +++ b/runner/options.go @@ -146,6 +146,7 @@ type Options struct { filterStatusCode []int filterContentLength []int Output string + OutputAll bool StoreResponseDir string HTTPProxy string SocksProxy string @@ -372,6 +373,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), + flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), @@ -437,6 +439,15 @@ func ParseOptions() *Options { _ = flagSet.Parse() + if options.OutputAll && options.Output == "" { + gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all") + } + + if options.OutputAll { + options.JSONOutput = true + options.CSVOutput = true + } + if cfgFile != "" { if !fileutil.FileExists(cfgFile) { gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile) @@ -507,11 +518,6 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } - multiOutput := options.CSVOutput && options.JSONOutput - if multiOutput { - return fmt.Errorf("results can only be displayed in one format: 'JSON' or 'CSV'") - } - if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 87b50c8..b2382a5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -621,27 +621,49 @@ func (r *Runner) RunEnumeration() { } }() - var f, indexFile, indexScreenshotFile *os.File + var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File - if r.options.Output != "" { - var err error - if r.options.Resume { - f, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) - } else { - f, err = os.Create(r.options.Output) - } - if err != nil { - gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", r.options.Output, err) - } - defer f.Close() //nolint + if r.options.Output != "" && r.options.OutputAll { + plainFile = openOrCreateFile(r.options.Resume, r.options.Output) + defer plainFile.Close() + jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json") + defer jsonFile.Close() + csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv") + defer csvFile.Close() } + + jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput) + jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput) + if r.options.Output != "" && plainFile == nil && !jsonOrCsv { + plainFile = openOrCreateFile(r.options.Resume, r.options.Output) + defer plainFile.Close() + } + + if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil { + ext := "" + if jsonAndCsv { + ext = ".json" + } + jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) + defer jsonFile.Close() + } + + if r.options.Output != "" && r.options.CSVOutput && csvFile == nil { + ext := "" + if jsonAndCsv { + ext = ".csv" + } + csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext) + defer csvFile.Close() + } + if r.options.CSVOutput { outEncoding := strings.ToLower(r.options.CSVOutputEncoding) switch outEncoding { case "": // no encoding do nothing case "utf-8", "utf8": bomUtf8 := []byte{0xEF, 0xBB, 0xBF} - _, err := f.Write(bomUtf8) + _, err := csvFile.Write(bomUtf8) if err != nil { gologger.Fatal().Msgf("err on file write: %s\n", err) } @@ -649,10 +671,13 @@ func (r *Runner) RunEnumeration() { gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding) } header := Result{}.CSVHeader() - gologger.Silent().Msgf("%s\n", header) - if f != nil { + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", header) + } + + if csvFile != nil { //nolint:errcheck // this method needs a small refactor to reduce complexity - f.WriteString(header + "\n") + csvFile.WriteString(header + "\n") } } if r.options.StoreResponseDir != "" { @@ -859,18 +884,40 @@ func (r *Runner) RunEnumeration() { } } } - row := resp.str - if r.options.JSONOutput { - row = resp.JSON(&r.scanopts) - } else if r.options.CSVOutput { - row = resp.CSVRow(&r.scanopts) + if !jsonOrCsv || jsonAndCsv || r.options.OutputAll { + gologger.Silent().Msgf("%s\n", resp.str) } - gologger.Silent().Msgf("%s\n", row) - if f != nil { + //nolint:errcheck // this method needs a small refactor to reduce complexity + if plainFile != nil { + plainFile.WriteString(resp.str + "\n") + } + + if r.options.JSONOutput { + row := resp.JSON(&r.scanopts) + + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", row) + } + //nolint:errcheck // this method needs a small refactor to reduce complexity - f.WriteString(row + "\n") + if jsonFile != nil { + jsonFile.WriteString(row + "\n") + } + } + + if r.options.CSVOutput { + row := resp.CSVRow(&r.scanopts) + + if !r.options.OutputAll && !jsonAndCsv { + gologger.Silent().Msgf("%s\n", row) + } + + //nolint:errcheck // this method needs a small refactor to reduce complexity + if csvFile != nil { + csvFile.WriteString(row + "\n") + } } for _, nextStep := range nextSteps { @@ -975,6 +1022,20 @@ func (r *Runner) RunEnumeration() { wgoutput.Wait() } +func openOrCreateFile(resume bool, filename string) *os.File { + var err error + var f *os.File + if resume { + f, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + f, err = os.Create(filename) + } + if err != nil { + gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", filename, err) + } + return f +} + func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } @@ -1614,7 +1675,10 @@ retry: hashesMap := make(map[string]interface{}) if scanopts.Hashes != "" { hs := strings.Split(scanopts.Hashes, ",") - builder.WriteString(" [") + outputHashes := !(r.options.JSONOutput || r.options.OutputAll) + if outputHashes { + builder.WriteString(" [") + } for index, hashType := range hs { var ( hashHeader, hashBody string @@ -1643,17 +1707,21 @@ retry: if hashBody != "" { hashesMap[fmt.Sprintf("body_%s", hashType)] = hashBody hashesMap[fmt.Sprintf("header_%s", hashType)] = hashHeader - if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(hashBody).String()) - } else { - builder.WriteString(hashBody) - } - if index != len(hs)-1 { - builder.WriteString(",") + if outputHashes { + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(hashBody).String()) + } else { + builder.WriteString(hashBody) + } + if index != len(hs)-1 { + builder.WriteString(",") + } } } } - builder.WriteRune(']') + if outputHashes { + builder.WriteRune(']') + } } if scanopts.OutputLinesCount { builder.WriteString(" [")