From 2cffea48835ac2c02baa6ce5a1b89d8432dd7e72 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 15 Sep 2021 17:40:03 -0500 Subject: [PATCH 1/5] bug in sending post request fix --- runner/runner.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 40246ed..37506d2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -700,7 +701,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -712,11 +713,9 @@ retry: if scanopts.Unsafe { req.Header.Add("Connection", "close") } - resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - var requestDump []byte if scanopts.Unsafe { var errDump error @@ -726,8 +725,10 @@ retry: } } else { // Create a copy on the fly of the request body - bodyBytes, _ := req.BodyBytes() - req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + buf := new(bytes.Buffer) + buf.ReadFrom(req.Body) + bodyBytes := buf.Bytes() + req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { @@ -740,6 +741,7 @@ retry: req.Body = nil } } + resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) // fix the final output url fullURL := req.URL.String() From 7d4273befd18446c04eda06209492959b83af81e Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 16 Sep 2021 16:13:37 -0500 Subject: [PATCH 2/5] bug-fix while sending post request --- runner/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 37506d2..85efd84 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -726,7 +726,11 @@ retry: } else { // Create a copy on the fly of the request body buf := new(bytes.Buffer) - buf.ReadFrom(req.Body) + _, err := buf.ReadFrom(req.Body) + if err != nil { + gologger.Fatal().Msgf("Could not read from request body: %s\n", err) + } + bodyBytes := buf.Bytes() req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error From 28fda6fa060d7c4658df319959682f160c8d8965 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Fri, 17 Sep 2021 22:55:43 -0500 Subject: [PATCH 3/5] changes to adrress failed test cases --- runner/runner.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 85efd84..1bef1c0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io" "io/ioutil" "net" "net/http" @@ -701,7 +700,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -713,6 +712,7 @@ retry: if scanopts.Unsafe { req.Header.Add("Connection", "close") } + resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } @@ -725,14 +725,12 @@ retry: } } else { // Create a copy on the fly of the request body - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(req.Body) - if err != nil { - gologger.Fatal().Msgf("Could not read from request body: %s\n", err) + var bodyBytes []byte + if scanopts.RequestBody != "" { + req.ContentLength = int64(len(scanopts.RequestBody)) + req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + bodyBytes = []byte(scanopts.RequestBody) } - - bodyBytes := buf.Bytes() - req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { @@ -745,8 +743,6 @@ retry: req.Body = nil } } - resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) - // fix the final output url fullURL := req.URL.String() parsedURL, _ := urlutil.Parse(fullURL) From f1c3474327b1ee496b78e4a28ec675aff49b682b Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 20 Sep 2021 22:44:16 -0500 Subject: [PATCH 4/5] remove bodybytes --- runner/runner.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 1bef1c0..cba318d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -725,11 +725,9 @@ retry: } } else { // Create a copy on the fly of the request body - var bodyBytes []byte if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) - bodyBytes = []byte(scanopts.RequestBody) } var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) @@ -738,7 +736,7 @@ retry: } // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) - if len(bodyBytes) == 0 { + if len(scanopts.RequestBody) == 0 { req.ContentLength = 0 req.Body = nil } From a39845bd3961112ecb235d44b3a7882d47dabb6c Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 23 Sep 2021 13:09:47 -0500 Subject: [PATCH 5/5] Added regression test for POST request with body --- cmd/integration-test/http.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 17103bf..460b087 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -20,6 +21,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body + "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body } type standardHttpGet struct { @@ -184,3 +186,26 @@ func (h *issue363) Execute() error { } return nil } + +type issue400 struct{} + +func (h *issue400) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + data, _ := ioutil.ReadAll(r.Body) + fmt.Fprintf(w, "data received %s", data) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL+"/receive", debug, "-body 'a=b'", "-x POST", "-status-code") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +}