functional test

cmd/functional-test/main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var (
+	debug   = os.Getenv("DEBUG") == "true"
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary")
+	devHttpxBinary  = flag.String("dev", "", "Dev Branch Httpx Binary")
+	testcases      = flag.String("testcases", "", "Test cases file for Httpx functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	var target string
+	if len(parts) > 1 {
+		finalArgs = parts[2:]
+		target = parts[0]
+	}
+	mainOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *mainHttpxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run httpx main test")
+	}
+	devOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *devHttpxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run httpx dev test")
+	}
+	if len(mainOutput) == len(devOutput) {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}

cmd/functional-test/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Building functional-test binary'
+go build
+
+echo 'Building HTTPX binary from current branch'
+go build -o httpx_dev ../httpx
+
+echo 'Installing latest release of HTTPX'
+GO111MODULE=on go build -v github.com/projectdiscovery/httpx/cmd/httpx
+
+echo 'Starting HTTPX functional test'
+./functional-test -main ./httpx -dev ./httpx_dev -testcases testcases.txt

cmd/functional-test/test-data/raw-request.txt

@@ -0,0 +1 @@
+GET /search?q=test HTTP/2

cmd/functional-test/test-data/request.txt

@@ -0,0 +1 @@
+https://www.example.com

cmd/functional-test/testcases.txt

@@ -0,0 +1,79 @@
+www.example.com {{binary}}
+www.example.com {{binary}} -l test-data/request.txt
+www.example.com {{binary}} -request test-data/raw-request.txt
+www.example.com {{binary}} -status-code
+www.example.com {{binary}} -tech-detect
+www.example.com {{binary}} -content-length
+www.example.com {{binary}} -web-server
+www.example.com {{binary}} -content-type
+www.example.com {{binary}} -response-time
+www.example.com {{binary}} -title
+www.example.com {{binary}} -location
+www.example.com {{binary}} -method
+www.example.com {{binary}} -websocket
+www.example.com {{binary}} -ip 
+www.example.com {{binary}} -cname
+www.example.com {{binary}} -cdn
+www.example.com {{binary}} -probe
+www.example.com {{binary}} -no-fallback
+www.example.com {{binary}} -match-code 200, -mc 200
+www.example.com {{binary}} -match-length 100, -ml 100
+www.example.com {{binary}} -match-string xyz, -ms xyz
+www.example.com {{binary}} -match-regex bbb, -mr bbb
+www.example.com {{binary}} -extract-regex aaa, -er aaa
+www.example.com {{binary}} -filter-code 401, -fc 401
+www.example.com {{binary}} -filter-length 23, -fl 23
+www.example.com {{binary}} -filter-string
+www.example.com {{binary}} -filter-regex
+www.example.com {{binary}} -threads 50
+www.example.com {{binary}} -rate-limit 5
+www.example.com {{binary}} -tls-grab
+www.example.com {{binary}} -tls-probe
+www.example.com {{binary}} -csp-probe
+www.example.com {{binary}} -pipeline
+www.example.com {{binary}} -http2
+www.example.com {{binary}} -vhost
+www.example.com {{binary}} -ports
+www.example.com {{binary}} -path
+www.example.com {{binary}} -paths
+www.example.com {{binary}} -output test-data/request.txt
+www.example.com {{binary}} -store-response
+www.example.com {{binary}} -store-response-dir test-data/request.txt
+www.example.com {{binary}} -json
+www.example.com {{binary}} -include-response -json
+www.example.com {{binary}} -include-chain
+www.example.com {{binary}} -store-chain
+www.example.com {{binary}} -csv
+www.example.com {{binary}} -response-size-to-save 1028
+www.example.com {{binary}} -response-size-to-read 1028
+www.example.com {{binary}} -allow 127.0.0.1:8080
+www.example.com {{binary}} -deny 127.0.0.1:6089
+www.example.com {{binary}} -random-agent
+www.example.com {{binary}} -header
+www.example.com {{binary}} -proxy www.example.com
+www.example.com {{binary}} -unsafe
+www.example.com {{binary}} -resume
+www.example.com {{binary}} -no-color
+www.example.com {{binary}} -no-fallback-scheme
+www.example.com {{binary}} -follow-redirects
+www.example.com {{binary}} -follow-host-redirects
+www.example.com {{binary}} -max-redirects 10
+www.example.com {{binary}} -vhost-input
+www.example.com {{binary}} -x
+www.example.com {{binary}} -body
+www.example.com {{binary}} -stream
+www.example.com {{binary}} -skip-dedupe -stream
+www.example.com {{binary}} -silent
+www.example.com {{binary}} -verbose
+www.example.com {{binary}} -version
+www.example.com {{binary}} -debug
+www.example.com {{binary}} -debug-req
+www.example.com {{binary}} -debug-resp
+www.example.com {{binary}} -stats
+www.example.com {{binary}} -retries 0
+www.example.com {{binary}} -timeout 10
+www.example.com {{binary}} -max-host-error 30
+www.example.com {{binary}} -exclude-cdn
+
+
+

internal/testutils/integration.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"fmt"
 )
 
 // RunNucleiAndGetResults returns a list of results for a template
@@ -33,6 +34,31 @@ func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, e
 	}
 	return parts, nil
 }
+func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, args []string) ([]string, error) {
+	cmd := exec.Command("bash", "-c")
+	cmdLine := fmt.Sprintf(`echo %s | %s `, target, httpxBinary)
+	cmdLine += strings.Join(args, " ")
+	if debug {
+		cmdLine += " -debug"
+		cmd.Stderr = os.Stderr
+	} else {
+		cmdLine += " -silent"
+	}
+
+	cmd.Args = append(cmd.Args, cmdLine)
+	data, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	parts := []string{}
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts,nil
+}
 
 // TestCase is a single integration test case
 type TestCase interface {