admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support multiple output format (#1242)

Browse Source 
* support multiple output format

* output plain text when -oA

* separate the conditions

* fail output-all when it's not used with output

* update doc

* remove hashes for plain output when -hash isn't used

* add integration test

* use filepath

* fix no hash std output

* remove json/csv extension for json/csv only output

* add ext when both -csv and -json is used

* fix assertion

* commit to last commit

* print filenames

* no tmpdir

* remove files

* remove prints

* lowercase options

* test update

---------

Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
This commit is contained in:
Dogan Can Bakir
2023-07-17 13:39:10 +03:00
committed by GitHub
parent 924eb5bc0c
commit c3eb8897e7
5 changed files with 157 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -7,3 +7,5 @@ cmd/functional-test/httpx_dev
cmd/functional-test/functional-test
cmd/functional-test/httpx
cmd/functional-test/*.cfg
.devcontainer
README.md
+1
View File
@@ -164,6 +164,7 @@ UPDATE:
OUTPUT:
-o, -output string file to write output results
-oa, -output-all filename to write output results in all formats
-sr, -store-response store http response to output directory
-srd, -store-response-dir string store http response to custom directory
-csv store output in csv format
cmd/integration-test/http.go
+42
View File
@@ -5,10 +5,12 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/httpx/internal/testutils"
fileutil "github.com/projectdiscovery/utils/file"
)
var httpTestcases = map[string]testutils.TestCase{
@@ -30,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{
"Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}},
"Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}},
"Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}},
"Output All": &outputAll{},
}
type standardHttpGet struct {
@@ -377,3 +380,42 @@ func (h *outputFilterCondition) Execute() error {
}
return nil
}
type outputAll struct {
}
func (h *outputAll) Execute() error {
var ts *httptest.Server
router := httprouter.New()
router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(200)
fmt.Fprint(w, `{"status": "ok"}`)
}))
ts = httptest.NewServer(router)
defer ts.Close()
fileName := "test_output_all"
_, hErr := testutils.RunHttpxAndGetResults(ts.URL, false, []string{"-o", fileName, "-oa"}...)
if hErr != nil {
return hErr
}
expectedFiles := []string{fileName, fileName + ".json", fileName + ".csv"}
var actualFiles []string
for _, file := range expectedFiles {
if fileutil.FileExists(file) {
actualFiles = append(actualFiles, file)
}
}
if len(actualFiles) != 3 {
return errIncorrectResultsCount(actualFiles)
}
for _, file := range actualFiles {
_ = os.Remove(file)
}
return nil
}
runner/options.go
+11 -5
View File
@@ -146,6 +146,7 @@ type Options struct {
filterStatusCode []int
filterContentLength []int
Output string
OutputAll bool
StoreResponseDir string
HTTPProxy string
SocksProxy string
@@ -372,6 +373,7 @@ func ParseOptions() *Options {
flagSet.CreateGroup("output", "Output",
flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"),
flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"),
flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
@@ -437,6 +439,15 @@ func ParseOptions() *Options {
_ = flagSet.Parse()
if options.OutputAll && options.Output == "" {
gologger.Fatal().Msg("Please specify an output file using -o/-output when using -oa/-output-all")
}
if options.OutputAll {
options.JSONOutput = true
options.CSVOutput = true
}
if cfgFile != "" {
if !fileutil.FileExists(cfgFile) {
gologger.Fatal().Msgf("given config file '%s' does not exist", cfgFile)
@@ -507,11 +518,6 @@ func (options *Options) ValidateOptions() error {
return fmt.Errorf("file '%s' does not exist", options.InputRawRequest)
}
multiOutput := options.CSVOutput && options.JSONOutput
if multiOutput {
return fmt.Errorf("results can only be displayed in one format: 'JSON' or 'CSV'")
}
if options.Silent {
incompatibleFlagsList := flagsIncompatibleWithSilent(options)
if len(incompatibleFlagsList) > 0 {
runner/runner.go
+101 -33
View File
@@ -621,27 +621,49 @@ func (r *Runner) RunEnumeration() {
}
}()
var f, indexFile, indexScreenshotFile *os.File
var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File
if r.options.Output != "" {
var err error
if r.options.Resume {
f, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
f, err = os.Create(r.options.Output)
}
if err != nil {
gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", r.options.Output, err)
}
defer f.Close() //nolint
if r.options.Output != "" && r.options.OutputAll {
plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
defer plainFile.Close()
jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+".json")
defer jsonFile.Close()
csvFile = openOrCreateFile(r.options.Resume, r.options.Output+".csv")
defer csvFile.Close()
}
jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput)
jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput)
if r.options.Output != "" && plainFile == nil && !jsonOrCsv {
plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
defer plainFile.Close()
}
if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil {
ext := ""
if jsonAndCsv {
ext = ".json"
}
jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
defer jsonFile.Close()
}
if r.options.Output != "" && r.options.CSVOutput && csvFile == nil {
ext := ""
if jsonAndCsv {
ext = ".csv"
}
csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
defer csvFile.Close()
}
if r.options.CSVOutput {
outEncoding := strings.ToLower(r.options.CSVOutputEncoding)
switch outEncoding {
case "": // no encoding do nothing
case "utf-8", "utf8":
bomUtf8 := []byte{0xEF, 0xBB, 0xBF}
_, err := f.Write(bomUtf8)
_, err := csvFile.Write(bomUtf8)
if err != nil {
gologger.Fatal().Msgf("err on file write: %s\n", err)
}
@@ -649,10 +671,13 @@ func (r *Runner) RunEnumeration() {
gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding)
}
header := Result{}.CSVHeader()
gologger.Silent().Msgf("%s\n", header)
if f != nil {
if !r.options.OutputAll && !jsonAndCsv {
gologger.Silent().Msgf("%s\n", header)
}
if csvFile != nil {
//nolint:errcheck // this method needs a small refactor to reduce complexity
f.WriteString(header + "\n")
csvFile.WriteString(header + "\n")
}
}
if r.options.StoreResponseDir != "" {
@@ -859,18 +884,40 @@ func (r *Runner) RunEnumeration() {
}
}
}
row := resp.str
if r.options.JSONOutput {
row = resp.JSON(&r.scanopts)
} else if r.options.CSVOutput {
row = resp.CSVRow(&r.scanopts)
if !jsonOrCsv || jsonAndCsv || r.options.OutputAll {
gologger.Silent().Msgf("%s\n", resp.str)
}
gologger.Silent().Msgf("%s\n", row)
if f != nil {
//nolint:errcheck // this method needs a small refactor to reduce complexity
if plainFile != nil {
plainFile.WriteString(resp.str + "\n")
}
if r.options.JSONOutput {
row := resp.JSON(&r.scanopts)
if !r.options.OutputAll && !jsonAndCsv {
gologger.Silent().Msgf("%s\n", row)
}
//nolint:errcheck // this method needs a small refactor to reduce complexity
f.WriteString(row + "\n")
if jsonFile != nil {
jsonFile.WriteString(row + "\n")
}
}
if r.options.CSVOutput {
row := resp.CSVRow(&r.scanopts)
if !r.options.OutputAll && !jsonAndCsv {
gologger.Silent().Msgf("%s\n", row)
}
//nolint:errcheck // this method needs a small refactor to reduce complexity
if csvFile != nil {
csvFile.WriteString(row + "\n")
}
}
for _, nextStep := range nextSteps {
@@ -975,6 +1022,20 @@ func (r *Runner) RunEnumeration() {
wgoutput.Wait()
}
func openOrCreateFile(resume bool, filename string) *os.File {
var err error
var f *os.File
if resume {
f, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
f, err = os.Create(filename)
}
if err != nil {
gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", filename, err)
}
return f
}
func (r *Runner) GetScanOpts() ScanOptions {
return r.scanopts
}
@@ -1614,7 +1675,10 @@ retry:
hashesMap := make(map[string]interface{})
if scanopts.Hashes != "" {
hs := strings.Split(scanopts.Hashes, ",")
builder.WriteString(" [")
outputHashes := !(r.options.JSONOutput || r.options.OutputAll)
if outputHashes {
builder.WriteString(" [")
}
for index, hashType := range hs {
var (
hashHeader, hashBody string
@@ -1643,17 +1707,21 @@ retry:
if hashBody != "" {
hashesMap[fmt.Sprintf("body_%s", hashType)] = hashBody
hashesMap[fmt.Sprintf("header_%s", hashType)] = hashHeader
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(hashBody).String())
} else {
builder.WriteString(hashBody)
}
if index != len(hs)-1 {
builder.WriteString(",")
if outputHashes {
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(hashBody).String())
} else {
builder.WriteString(hashBody)
}
if index != len(hs)-1 {
builder.WriteString(",")
}
}
}
}
builder.WriteRune(']')
if outputHashes {
builder.WriteRune(']')
}
}
if scanopts.OutputLinesCount {
builder.WriteString(" [")