diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go new file mode 100644 index 0000000..52a3db3 --- /dev/null +++ b/cmd/functional-test/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + + "github.com/projectdiscovery/httpx/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[✓]").String() + failed = aurora.Red("[✘]").String() + errored = false + + mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary") + devHttpxBinary = flag.String("dev", "", "Dev Branch Httpx Binary") + testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests") +) + +func main() { + flag.Parse() + + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + file, err := os.Open(*testcases) + if err != nil { + return errors.Wrap(err, "could not open test cases") + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + if err := runIndividualTestCase(text); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, text) + } + } + return nil +} + +func runIndividualTestCase(testcase string) error { + parts := strings.Fields(testcase) + + var finalArgs []string + var target string + if len(parts) > 1 { + finalArgs = parts[2:] + target = parts[0] + } + mainOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *mainHttpxBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run httpx main test") + } + devOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *devHttpxBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run httpx dev test") + } + if len(mainOutput) == len(devOutput) { + return nil + } + return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) +} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh new file mode 100755 index 0000000..be5e8c7 --- /dev/null +++ b/cmd/functional-test/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo 'Building functional-test binary' +go build + +echo 'Building HTTPX binary from current branch' +go build -o httpx_dev ../httpx + +echo 'Installing latest release of HTTPX' +GO111MODULE=on go build -v github.com/projectdiscovery/httpx/cmd/httpx + +echo 'Starting HTTPX functional test' +./functional-test -main ./httpx -dev ./httpx_dev -testcases testcases.txt diff --git a/cmd/functional-test/test-data/raw-request.txt b/cmd/functional-test/test-data/raw-request.txt new file mode 100644 index 0000000..94b072c --- /dev/null +++ b/cmd/functional-test/test-data/raw-request.txt @@ -0,0 +1 @@ +GET /search?q=test HTTP/2 diff --git a/cmd/functional-test/test-data/request.txt b/cmd/functional-test/test-data/request.txt new file mode 100644 index 0000000..284d3f2 --- /dev/null +++ b/cmd/functional-test/test-data/request.txt @@ -0,0 +1 @@ +https://www.example.com diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt new file mode 100644 index 0000000..90c4f69 --- /dev/null +++ b/cmd/functional-test/testcases.txt @@ -0,0 +1,79 @@ +www.example.com {{binary}} +www.example.com {{binary}} -l test-data/request.txt +www.example.com {{binary}} -request test-data/raw-request.txt +www.example.com {{binary}} -status-code +www.example.com {{binary}} -tech-detect +www.example.com {{binary}} -content-length +www.example.com {{binary}} -web-server +www.example.com {{binary}} -content-type +www.example.com {{binary}} -response-time +www.example.com {{binary}} -title +www.example.com {{binary}} -location +www.example.com {{binary}} -method +www.example.com {{binary}} -websocket +www.example.com {{binary}} -ip +www.example.com {{binary}} -cname +www.example.com {{binary}} -cdn +www.example.com {{binary}} -probe +www.example.com {{binary}} -no-fallback +www.example.com {{binary}} -match-code 200, -mc 200 +www.example.com {{binary}} -match-length 100, -ml 100 +www.example.com {{binary}} -match-string xyz, -ms xyz +www.example.com {{binary}} -match-regex bbb, -mr bbb +www.example.com {{binary}} -extract-regex aaa, -er aaa +www.example.com {{binary}} -filter-code 401, -fc 401 +www.example.com {{binary}} -filter-length 23, -fl 23 +www.example.com {{binary}} -filter-string +www.example.com {{binary}} -filter-regex +www.example.com {{binary}} -threads 50 +www.example.com {{binary}} -rate-limit 5 +www.example.com {{binary}} -tls-grab +www.example.com {{binary}} -tls-probe +www.example.com {{binary}} -csp-probe +www.example.com {{binary}} -pipeline +www.example.com {{binary}} -http2 +www.example.com {{binary}} -vhost +www.example.com {{binary}} -ports +www.example.com {{binary}} -path +www.example.com {{binary}} -paths +www.example.com {{binary}} -output test-data/request.txt +www.example.com {{binary}} -store-response +www.example.com {{binary}} -store-response-dir test-data/request.txt +www.example.com {{binary}} -json +www.example.com {{binary}} -include-response -json +www.example.com {{binary}} -include-chain +www.example.com {{binary}} -store-chain +www.example.com {{binary}} -csv +www.example.com {{binary}} -response-size-to-save 1028 +www.example.com {{binary}} -response-size-to-read 1028 +www.example.com {{binary}} -allow 127.0.0.1:8080 +www.example.com {{binary}} -deny 127.0.0.1:6089 +www.example.com {{binary}} -random-agent +www.example.com {{binary}} -header +www.example.com {{binary}} -proxy www.example.com +www.example.com {{binary}} -unsafe +www.example.com {{binary}} -resume +www.example.com {{binary}} -no-color +www.example.com {{binary}} -no-fallback-scheme +www.example.com {{binary}} -follow-redirects +www.example.com {{binary}} -follow-host-redirects +www.example.com {{binary}} -max-redirects 10 +www.example.com {{binary}} -vhost-input +www.example.com {{binary}} -x +www.example.com {{binary}} -body +www.example.com {{binary}} -stream +www.example.com {{binary}} -skip-dedupe -stream +www.example.com {{binary}} -silent +www.example.com {{binary}} -verbose +www.example.com {{binary}} -version +www.example.com {{binary}} -debug +www.example.com {{binary}} -debug-req +www.example.com {{binary}} -debug-resp +www.example.com {{binary}} -stats +www.example.com {{binary}} -retries 0 +www.example.com {{binary}} -timeout 10 +www.example.com {{binary}} -max-host-error 30 +www.example.com {{binary}} -exclude-cdn + + + diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index 6f7b28b..d058952 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "strings" + "fmt" ) // RunNucleiAndGetResults returns a list of results for a template @@ -33,6 +34,31 @@ func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, e } return parts, nil } +func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, httpxBinary) + cmdLine += strings.Join(args, " ") + if debug { + cmdLine += " -debug" + cmd.Stderr = os.Stderr + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts,nil +} // TestCase is a single integration test case type TestCase interface {