diff --git a/.github/workflows/flamegraph.yaml b/.github/workflows/flamegraph.yaml index b182af56b..c67099c4f 100644 --- a/.github/workflows/flamegraph.yaml +++ b/.github/workflows/flamegraph.yaml @@ -14,8 +14,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - run: make build - name: "Setup environment (push)" diff --git a/.github/workflows/generate-pgo.yaml b/.github/workflows/generate-pgo.yaml index ea1b305bb..ec1325d04 100644 --- a/.github/workflows/generate-pgo.yaml +++ b/.github/workflows/generate-pgo.yaml @@ -20,8 +20,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - run: | for i in {1..${{ env.TARGET_COUNT }}}; do echo "${{ env.TARGET }}/-/?_=${i}" >> "${{ env.TARGET_LIST }}"; diff --git a/.github/workflows/perf-regression.yaml b/.github/workflows/perf-regression.yaml index b0f362a35..286b595c0 100644 --- a/.github/workflows/perf-regression.yaml +++ b/.github/workflows/perf-regression.yaml @@ -13,8 +13,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - run: make build-test - run: ./bin/nuclei.test -test.run - -test.bench=. -test.benchmem ./cmd/nuclei/ | tee $BENCH_OUT env: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b4a7abef5..3aaed3013 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -24,7 +24,6 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - uses: projectdiscovery/actions/golangci-lint/v2@v1 tests: @@ -40,8 +39,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - uses: projectdiscovery/actions/free-disk-space@v1 with: llvm: 'false' @@ -56,7 +54,7 @@ jobs: - run: make test env: PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" - - run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version + - run: go run -race . -l ${{ github.workspace }}/internal/tests/functional/testdata/targets.txt -id tech-detect,tls-version if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099 working-directory: cmd/nuclei/ @@ -69,8 +67,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - name: "Simple" run: go run . working-directory: examples/simple/ @@ -94,15 +91,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 - - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 - uses: projectdiscovery/actions/setup/python@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - run: bash run.sh "${{ matrix.os }}" + - uses: projectdiscovery/actions/setup/go@v1 + - run: make integration env: PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}" timeout-minutes: 50 - working-directory: integration_tests/ functional: name: "Functional tests" @@ -116,12 +110,13 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 - - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/nuclei@v1 - uses: projectdiscovery/actions/setup/python@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 - - run: bash run.sh - working-directory: cmd/functional-test/ + - uses: projectdiscovery/actions/setup/go@v1 + - uses: projectdiscovery/nuclei-action@v3 + with: + version: latest + install-only: true + - run: make functional validate: name: "Template validate" @@ -132,7 +127,7 @@ jobs: steps: - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - - uses: projectdiscovery/actions/cache/go-rod-browser@v1 + - uses: projectdiscovery/nuclei-action/cache@v3 - run: make template-validate codeql: diff --git a/.gitignore b/.gitignore index 9963fdd0b..f8b8c2d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,7 @@ /scrapefunc /scrapefuncs /tsgen -/integration_tests/integration-test -/integration_tests/nuclei +*.test # Templates /*.yaml diff --git a/.run/DSLFunctionsIT.run.xml b/.run/DSLFunctionsIT.run.xml deleted file mode 100644 index 49cf9c584..000000000 --- a/.run/DSLFunctionsIT.run.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.run/IntegrationTests.run.xml b/.run/IntegrationTests.run.xml deleted file mode 100644 index dad3e3343..000000000 --- a/.run/IntegrationTests.run.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - \ No newline at end of file diff --git a/.run/RegressionTests.run.xml b/.run/RegressionTests.run.xml deleted file mode 100644 index 70229deba..000000000 --- a/.run/RegressionTests.run.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 605ed9f4e..5a1c98e32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,8 +11,8 @@ Nuclei is a modern, high-performance vulnerability scanner built in Go that leve ### Building and Testing - `make build` - Build the main nuclei binary to ./bin/nuclei - `make test` - Run unit tests with race detection -- `make integration` - Run integration tests (builds and runs test suite) -- `make functional` - Run functional tests +- `make integration` - Run the native integration suite via `go test -tags=integration ./internal/tests/integration` +- `make functional` - CI-only functional suite entry point - `make vet` - Run go vet for code analysis - `make tidy` - Clean up go modules @@ -29,7 +29,7 @@ Nuclei is a modern, high-performance vulnerability scanner built in Go that leve ### Testing Specific Components - Run single test: `go test -v ./pkg/path/to/package -run TestName` -- Integration tests are in `integration_tests/` and can be run via `make integration` +- Integration tests are in `internal/tests/integration/` and run via `go test -tags=integration ./internal/tests/integration` ## Architecture Overview @@ -77,7 +77,8 @@ Each protocol (HTTP, DNS, Network, etc.) implements: ## Key Directories - **lib/** - SDK for embedding nuclei as a library - **examples/** - Usage examples for different scenarios -- **integration_tests/** - Integration test suite with protocol-specific tests +- **internal/tests/integration/** - Native integration suite and owned testdata for harness coverage +- **internal/tests/functional/** - CI-only native functional comparison suite and testcase assets - **pkg/fuzz/** - Fuzzing engine and DAST capabilities - **pkg/input/** - Input processing for various formats (Burp, OpenAPI, etc.) - **pkg/reporting/** - Result export and issue tracking integrations \ No newline at end of file diff --git a/DESIGN.md b/DESIGN.md index b1d713401..522e0cf49 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -454,7 +454,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti } ``` -That's it, you've added a new protocol to Nuclei. The next good step would be to write integration tests which are described in `integration-tests` and `cmd/integration-tests` directories. +That's it, you've added a new protocol to Nuclei. The next good step would be to add native integration coverage under `internal/tests/integration` and run it with `go test -tags=integration ./internal/tests/integration`. ## Profiling and Tracing diff --git a/Makefile b/Makefile index 2479491f6..f9e042bb0 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ GOCMD := go GOBUILD := $(GOCMD) build GOBUILD_OUTPUT := +GOBUILD_OUTPUT_EXT := GOBUILD_PACKAGES := GOBUILD_ADDITIONAL_ARGS := GOMOD := $(GOCMD) mod @@ -16,22 +17,26 @@ ifneq ($(shell go env GOOS),darwin) LDFLAGS += -extldflags "-static" endif +ifeq ($(shell go env GOOS),windows) + GOBUILD_OUTPUT_EXT := .exe +endif + .PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs fuzz fuzz-ci fuzz-tools .PHONY: devtools-tsgen docs docgen dsl-docs functional go-build lint lint-strict fuzzplayground syntax-docs -.PHONY: integration jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint +.PHONY: integration integration-debug jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint .PHONY: tidy ts verify download vet template-validate build-fuzz discover-fuzz-packages all: build clean: - rm -f '${GOBUILD_OUTPUT}' 2>/dev/null + rm -f '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' 2>/dev/null go-build: clean go-build: CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \ - -o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES) + -o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES) -build: GOFLAGS = -v -pgo=auto +build: GOFLAGS = -pgo=auto build: GOBUILD_OUTPUT = ./bin/nuclei build: GOBUILD_PACKAGES = cmd/nuclei/main.go build: go-build @@ -42,7 +47,7 @@ build-test: GOBUILD_PACKAGES = ./cmd/nuclei/ build-test: clean build-test: CGO_ENABLED=0 $(GOCMD) test -c -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \ - -o '${GOBUILD_OUTPUT}' ${GOBUILD_PACKAGES} + -o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' ${GOBUILD_PACKAGES} build-stats: GOBUILD_OUTPUT = ./bin/nuclei-stats build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go @@ -77,15 +82,24 @@ syntax-docs: docgen syntax-docs: ./bin/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json -test: GOFLAGS = -race -v -timeout 30m -count 1 +test: GOFLAGS = -race -v -timeout 1h -count 1 test: $(GOTEST) $(GOFLAGS) ./... integration: - cd integration_tests; bash run.sh + $(GOTEST) -tags=integration -timeout 1h ./internal/tests/integration -functional: - cd cmd/functional-test; bash run.sh +integration-debug: + $(GOTEST) -tags=integration ./internal/tests/integration -v $(GO_TEST_ARGS) -args $(INTEGRATION_ARGS) + +functional: build + @release_binary="$$(command -v nuclei.exe 2>/dev/null || command -v nuclei 2>/dev/null)"; \ + if [ -z "$$release_binary" ]; then \ + echo "release nuclei binary not found on PATH"; \ + exit 1; \ + fi; \ + RELEASE_BINARY="$$release_binary" DEV_BINARY="$(PWD)/bin/nuclei" \ + $(GOTEST) -tags=functional -timeout 1h ./internal/tests/functional tidy: $(GOMOD) tidy diff --git a/_typos.toml b/_typos.toml index a2b142a5f..1613d5153 100644 --- a/_typos.toml +++ b/_typos.toml @@ -9,13 +9,13 @@ extend-exclude = [ "README_PT-BR.md", "README_TR.md", # Test fixtures and data files - "integration_tests/", + "internal/tests/integration/testdata/", "pkg/input/formats/testdata/", "pkg/output/stats/waf/", # Deserialization test data "pkg/protocols/common/helpers/deserialization/testdata/", # Certificate/encoded data in test utilities - "pkg/testutils/integration.go", + "internal/tests/testutils/integration.go", # Vendor directory "vendor/", # Go checksum file diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go deleted file mode 100644 index caab15f2a..000000000 --- a/cmd/functional-test/main.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "log" - "os" - "strings" - - "github.com/kitabisa/go-ci" - "github.com/logrusorgru/aurora" - "github.com/pkg/errors" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var ( - success = aurora.Green("[✓]").String() - failed = aurora.Red("[✘]").String() - - mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary") - devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary") - testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests") -) - -func main() { - flag.Parse() - - debug := os.Getenv("DEBUG") == "true" || os.Getenv("RUNNER_DEBUG") == "1" - - if err, errored := runFunctionalTests(debug); err != nil { - log.Fatalf("Could not run functional tests: %s\n", err) - } else if errored { - os.Exit(1) - } -} - -func runFunctionalTests(debug bool) (error, bool) { - file, err := os.Open(*testcases) - if err != nil { - return errors.Wrap(err, "could not open test cases"), true - } - defer func() { - _ = file.Close() - }() - - errored, failedTestCases := runTestCases(file, debug) - - if ci.IsCI() { - fmt.Println("::group::Failed tests with debug") - for _, failedTestCase := range failedTestCases { - _ = runTestCase(failedTestCase, true) - } - fmt.Println("::endgroup::") - } - - return nil, errored -} - -func runTestCases(file *os.File, debug bool) (bool, []string) { - errored := false - var failedTestCases []string - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - testCase := strings.TrimSpace(scanner.Text()) - if testCase == "" { - continue - } - // skip comments - if strings.HasPrefix(testCase, "#") { - continue - } - if runTestCase(testCase, debug) { - errored = true - failedTestCases = append(failedTestCases, testCase) - } - } - return errored, failedTestCases -} - -func runTestCase(testCase string, debug bool) bool { - if err := runIndividualTestCase(testCase, debug); err != nil { - fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testCase, err) - return true - } else { - fmt.Printf("%s Test \"%s\" passed!\n", success, testCase) - } - return false -} - -func runIndividualTestCase(testcase string, debug bool) error { - quoted := false - - // split upon unquoted spaces - parts := strings.FieldsFunc(testcase, func(r rune) bool { - if r == '"' { - quoted = !quoted - } - return !quoted && r == ' ' - }) - - // Quoted strings containing spaces are expressions and must have trailing \" removed - for index, part := range parts { - if strings.Contains(part, " ") { - parts[index] = strings.Trim(part, "\"") - } - } - - var finalArgs []string - if len(parts) > 1 { - finalArgs = parts[1:] - } - mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, debug, finalArgs) - if err != nil { - return errors.Wrap(err, "could not run nuclei main test") - } - devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, debug, finalArgs) - if err != nil { - return errors.Wrap(err, "could not run nuclei dev test") - } - if mainOutput == devOutput { - return nil - } - return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) -} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh deleted file mode 100755 index a955ad8a7..000000000 --- a/cmd/functional-test/run.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -if [ "${RUNNER_OS}" == "Windows" ]; then - EXT=".exe" -elif [ "${RUNNER_OS}" == "macOS" ]; then - if [ "${CI}" == "true" ]; then - sudo sysctl -w kern.maxfiles{,perproc}=524288 - sudo launchctl limit maxfiles 65536 524288 - fi - - ORIGINAL_ULIMIT="$(ulimit -n)" - ulimit -n 65536 || true -fi - -mkdir -p .nuclei-config/nuclei/ -touch .nuclei-config/nuclei/.nuclei-ignore - -echo "::group::Building functional-test binary" -go build -o "functional-test${EXT}" -echo "::endgroup::" - -echo "::group::Building Nuclei binary from current branch" -go build -o "nuclei-dev${EXT}" ../nuclei -echo "::endgroup::" - -echo "::group::Building latest release of nuclei" -go build -o "nuclei${EXT}" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei -echo "::endgroup::" - -echo "::group::Installing nuclei templates" -eval "./nuclei-dev${EXT} -update-templates" -echo "::endgroup::" - -echo "::group::Validating templates" -eval "./nuclei-dev${EXT} -validate" -echo "::endgroup::" - -echo "Starting Nuclei functional test" -eval "./functional-test${EXT} -main ./nuclei${EXT} -dev ./nuclei-dev${EXT} -testcases testcases.txt" - -if [ "${RUNNER_OS}" == "macOS" ]; then - ulimit -n "${ORIGINAL_ULIMIT}" || true -fi diff --git a/cmd/integration-test/custom-dir.go b/cmd/integration-test/custom-dir.go deleted file mode 100644 index b1ea83cc6..000000000 --- a/cmd/integration-test/custom-dir.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "os" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -type customConfigDirTest struct{} - -var customConfigDirTestCases = []TestCaseInfo{ - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &customConfigDirTest{}}, -} - -// Execute executes a test case and returns an error if occurred -func (h *customConfigDirTest) Execute(filePath string) error { - customTempDirectory, err := os.MkdirTemp("", "") - if err != nil { - return err - } - defer func() { - _ = os.RemoveAll(customTempDirectory) - }() - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDirectory}, "-t", filePath, "-u", "8x8exch02.8x8.com") - if err != nil { - return err - } - if len(results) == 0 { - return nil - } - files, err := os.ReadDir(customTempDirectory) - if err != nil { - return err - } - var fileNames []string - for _, file := range files { - fileNames = append(fileNames, file.Name()) - } - return expectResultsCount(fileNames, 4) -} diff --git a/cmd/integration-test/dns.go b/cmd/integration-test/dns.go deleted file mode 100644 index 82cf9c91c..000000000 --- a/cmd/integration-test/dns.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var dnsTestCases = []TestCaseInfo{ - {Path: "protocols/dns/a.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/aaaa.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/cname.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/srv.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/ns.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/txt.yaml", TestCase: &dnsBasic{}}, - {Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}}, - {Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}}, - {Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}}, - {Path: "protocols/dns/variables.yaml", TestCase: &dnsVariables{}}, - {Path: "protocols/dns/payload.yaml", TestCase: &dnsPayload{}}, - {Path: "protocols/dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}}, -} - -type dnsBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsBasic) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type dnsPtr struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsPtr) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "1.1.1.1", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type dnsCAA struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsCAA) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type dnsTLSA struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsTLSA) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 0) -} - -type dnsVariables struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsVariables) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type dnsPayload struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsPayload) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug) - if err != nil { - return err - } - if err := expectResultsCount(results, 3); err != nil { - return err - } - - // override payload from CLI - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug, "-var", "subdomain_wordlist=subdomains.txt") - if err != nil { - return err - } - return expectResultsCount(results, 4) -} - -type dnsDSLMatcherVariable struct{} - -// Execute executes a test case and returns an error if occurred -func (h *dnsDSLMatcherVariable) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/file.go b/cmd/integration-test/file.go deleted file mode 100644 index 490bfe739..000000000 --- a/cmd/integration-test/file.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var fileTestcases = []TestCaseInfo{ - {Path: "protocols/file/matcher-with-or.yaml", TestCase: &fileWithOrMatcher{}}, - {Path: "protocols/file/matcher-with-and.yaml", TestCase: &fileWithAndMatcher{}}, - {Path: "protocols/file/matcher-with-nested-and.yaml", TestCase: &fileWithAndMatcher{}}, - {Path: "protocols/file/extract.yaml", TestCase: &fileWithExtractor{}}, -} - -type fileWithOrMatcher struct{} - -// Execute executes a test case and returns an error if occurred -func (h *fileWithOrMatcher) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type fileWithAndMatcher struct{} - -// Execute executes a test case and returns an error if occurred -func (h *fileWithAndMatcher) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type fileWithExtractor struct{} - -// Execute executes a test case and returns an error if occurred -func (h *fileWithExtractor) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/flow.go b/cmd/integration-test/flow.go deleted file mode 100644 index 1c103b95a..000000000 --- a/cmd/integration-test/flow.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "encoding/base64" - "fmt" - "net/http" - "net/http/httptest" - - "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var flowTestcases = []TestCaseInfo{ - {Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, - {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, - {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, - {Path: "flow/iterate-one-value-flow.yaml", TestCase: &iterateOneValueFlow{}}, - {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, - {Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}}, -} - -type conditionalFlow struct{} - -func (t *conditionalFlow) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "cloud.projectdiscovery.io", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type conditionalFlowNegative struct{} - -func (t *conditionalFlowNegative) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 0) -} - -type iterateValuesFlow struct{} - -func (t *iterateValuesFlow) Execute(filePath string) error { - router := httprouter.New() - testemails := []string{ - "secrets@scanme.sh", - "superadmin@scanme.sh", - } - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprint(w, testemails) - }) - router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Welcome ! This is test matcher text")) - }) - - router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("Welcome ! This is test matcher text")) - }) - - ts := httptest.NewServer(router) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - return expectResultsCount(results, 2) -} - -type iterateOneValueFlow struct{} - -func (t *iterateOneValueFlow) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type dnsNsProbe struct{} - -func (t *dnsNsProbe) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug) - if err != nil { - return err - } - return expectResultsCount(results, 2) -} - -func getBase64(input string) string { - return base64.StdEncoding.EncodeToString([]byte(input)) -} - -type flowHideMatcher struct{} - -func (t *flowHideMatcher) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - // this matcher should not return any results - return expectResultsCount(results, 0) -} diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go deleted file mode 100644 index d64d242f5..000000000 --- a/cmd/integration-test/integration-test.go +++ /dev/null @@ -1,285 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "regexp" - "runtime" - "slices" - "strings" - - "github.com/kitabisa/go-ci" - "github.com/logrusorgru/aurora" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground" - sliceutil "github.com/projectdiscovery/utils/slice" -) - -type TestCaseInfo struct { - Path string - TestCase testutils.TestCase - DisableOn func() bool -} - -var ( - debug = isDebugMode() - customTests = os.Getenv("TESTS") - protocol = os.Getenv("PROTO") - - success = aurora.Green("[✓]").String() - failed = aurora.Red("[✘]").String() - - protocolTests = map[string][]TestCaseInfo{ - "http": httpTestcases, - "interactsh": interactshTestCases, - "network": networkTestcases, - "dns": dnsTestCases, - "workflow": workflowTestcases, - "loader": loaderTestcases, - "profile-loader": profileLoaderTestcases, - "websocket": websocketTestCases, - "headless": headlessTestcases, - "whois": whoisTestCases, - "ssl": sslTestcases, - "library": libraryTestcases, - "templatesPath": templatesPathTestCases, - "templatesDir": templatesDirTestCases, - "env_vars": templatesDirEnvTestCases, - "file": fileTestcases, - "offlineHttp": offlineHttpTestcases, - "customConfigDir": customConfigDirTestCases, - "fuzzing": fuzzingTestCases, - "code": codeTestCases, - "multi": multiProtoTestcases, - "generic": genericTestcases, - "dsl": dslTestcases, - "flow": flowTestcases, - "javascript": jsTestcases, - "matcher-status": matcherStatusTestcases, - "exporters": exportersTestCases, - } - - // flakyTests are run with a retry count of 3 - flakyTests = map[string]bool{ - "protocols/http/self-contained-file-input.yaml": true, - } - - // For debug purposes - runProtocol = "" - runTemplate = "" - extraArgs = []string{} - interactshRetryCount = 3 -) - -func main() { - flag.StringVar(&runProtocol, "protocol", "", "run integration tests of given protocol") - flag.StringVar(&runTemplate, "template", "", "run integration test of given template") - flag.Parse() - - // allows passing extra args to nuclei - eargs := os.Getenv("DebugExtraArgs") - if eargs != "" { - extraArgs = strings.Split(eargs, " ") - testutils.ExtraDebugArgs = extraArgs - } - - if runProtocol != "" { - debugTests() - os.Exit(1) - } - - // start fuzz playground server - server := fuzzplayground.GetPlaygroundServer() - defer func() { - fuzzplayground.Cleanup() - _ = server.Close() - }() - - go func() { - if err := server.Start("localhost:8082"); err != nil { - if !strings.Contains(err.Error(), "Server closed") { - gologger.Fatal().Msgf("Could not start server: %s\n", err) - } - } - }() - - customTestsList := normalizeSplit(customTests) - failedTestTemplatePaths := runTests(customTestsList) - - if len(failedTestTemplatePaths) > 0 { - if ci.IsCI() { - // run failed tests again assuming they are flaky - // if they fail as well only then we assume that there is an actual issue - fmt.Println("::group::Running failed tests again") - failedTestTemplatePaths = runTests(failedTestTemplatePaths) - fmt.Println("::endgroup::") - - if len(failedTestTemplatePaths) > 0 { - debug = true - fmt.Println("::group::Failed integration tests in debug mode") - _ = runTests(failedTestTemplatePaths) - fmt.Println("::endgroup::") - } else { - fmt.Println("::group::All tests passed") - fmt.Println("::endgroup::") - os.Exit(0) - } - } - - os.Exit(1) - } -} - -// isDebugMode checks if debug mode is enabled via any of the supported debug -// environment variables. -func isDebugMode() bool { - debugEnvVars := []string{ - "DEBUG", - "ACTIONS_RUNNER_DEBUG", // GitHub Actions runner debug - // Add more debug environment variables here as needed - } - - truthyValues := []string{"true", "1", "yes", "on", "enabled"} - - for _, envVar := range debugEnvVars { - envValue := strings.ToLower(strings.TrimSpace(os.Getenv(envVar))) - if slices.Contains(truthyValues, envValue) { - return true - } - } - - return false -} - -// execute a testcase with retry and consider best of N -// intended for flaky tests like interactsh -func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) { - var err error - for i := 0; i < retryCount; i++ { - err = testCase.Execute(templatePath) - if err == nil { - fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath) - return "", nil - } - } - _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err) - return templatePath, err -} - -func debugTests() { - testCaseInfos := protocolTests[runProtocol] - for _, testCaseInfo := range testCaseInfos { - if (runTemplate != "" && !strings.Contains(testCaseInfo.Path, runTemplate)) || - (testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn()) { - continue - } - if runProtocol == "interactsh" { - if _, err := executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount); err != nil { - fmt.Printf("\n%v", err.Error()) - } - } else { - if _, err := execute(testCaseInfo.TestCase, testCaseInfo.Path); err != nil { - fmt.Printf("\n%v", err.Error()) - } - } - } -} - -func runTests(customTemplatePaths []string) []string { - var failedTestTemplatePaths []string - - for proto, testCaseInfos := range protocolTests { - if protocol != "" { - if !strings.EqualFold(proto, protocol) { - continue - } - } - if len(customTemplatePaths) == 0 { - fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto)) - } - for _, testCaseInfo := range testCaseInfos { - if testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn() { - fmt.Printf("skipping test case %v. disabled on %v.\n", aurora.Blue(testCaseInfo.Path), runtime.GOOS) - continue - } - if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, testCaseInfo.Path) { - var failedTemplatePath string - var err error - if proto == "interactsh" || strings.Contains(testCaseInfo.Path, "interactsh") { - failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount) - } else if flakyTests[testCaseInfo.Path] { - failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount) - } else { - failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path) - } - if err != nil { - failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath) - } - } - } - } - - return failedTestTemplatePaths -} - -func execute(testCase testutils.TestCase, templatePath string) (string, error) { - if err := testCase.Execute(templatePath); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, templatePath, err) - return templatePath, err - } - - fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath) - return "", nil -} - -func expectResultsCount(results []string, expectedNumbers ...int) error { - results = filterLines(results) - match := sliceutil.Contains(expectedNumbers, len(results)) - if !match { - return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) // nolint:all - } - return nil -} - -func normalizeSplit(str string) []string { - return strings.FieldsFunc(str, func(r rune) bool { - return r == ',' - }) -} - -// filterLines applies all filtering functions to the results -func filterLines(results []string) []string { - results = filterHeadlessLogs(results) - results = filterUnsignedTemplatesWarnings(results) - return results -} - -// if chromium is not installed go-rod installs it in .cache directory -// this function filters out the logs from download and installation -func filterHeadlessLogs(results []string) []string { - // [launcher.Browser] 2021/09/23 15:24:05 [launcher] [info] Starting browser - filtered := []string{} - for _, result := range results { - if strings.Contains(result, "[launcher.Browser]") { - continue - } - filtered = append(filtered, result) - } - return filtered -} - -// filterUnsignedTemplatesWarnings filters out warning messages about unsigned templates -func filterUnsignedTemplatesWarnings(results []string) []string { - filtered := []string{} - unsignedTemplatesRegex := regexp.MustCompile(`Loading \d+ unsigned templates for scan\. Use with caution\.`) - for _, result := range results { - if unsignedTemplatesRegex.MatchString(result) { - continue - } - filtered = append(filtered, result) - } - return filtered -} diff --git a/cmd/integration-test/javascript.go b/cmd/integration-test/javascript.go deleted file mode 100644 index 6cc71e91a..000000000 --- a/cmd/integration-test/javascript.go +++ /dev/null @@ -1,502 +0,0 @@ -package main - -import ( - "log" - "time" - - "github.com/ory/dockertest/v3" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - osutils "github.com/projectdiscovery/utils/os" - "go.uber.org/multierr" -) - -var jsTestcases = []TestCaseInfo{ - {Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}}, - {Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}}, - {Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/oracle-auth-test.yaml", TestCase: &javascriptOracleAuthTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}}, - {Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/mysql-connect.yaml", TestCase: &javascriptMySQLConnect{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, - {Path: "protocols/javascript/multi-ports.yaml", TestCase: &javascriptMultiPortsSSH{}}, - {Path: "protocols/javascript/no-port-args.yaml", TestCase: &javascriptNoPortArgs{}}, - {Path: "protocols/javascript/telnet-auth-test.yaml", TestCase: &javascriptTelnetAuthTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, -} - -var ( - redisResource *dockertest.Resource - sshResource *dockertest.Resource - oracleResource *dockertest.Resource - vncResource *dockertest.Resource - telnetResource *dockertest.Resource - postgresResource *dockertest.Resource - mysqlResource *dockertest.Resource - rsyncResource *dockertest.Resource - pool *dockertest.Pool - defaultRetry = 3 -) - -type javascriptNetHttps struct{} - -func (j *javascriptNetHttps) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type javascriptRedisPassBrute struct{} - -func (j *javascriptRedisPassBrute) Execute(filePath string) error { - if redisResource == nil || pool == nil { - // skip test as redis is not running - return nil - } - tempPort := redisResource.GetPort("6379/tcp") - finalURL := "localhost:" + tempPort - defer purge(redisResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let ssh server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptSSHServerFingerprint struct{} - -func (j *javascriptSSHServerFingerprint) Execute(filePath string) error { - if sshResource == nil || pool == nil { - // skip test as redis is not running - return nil - } - tempPort := sshResource.GetPort("2222/tcp") - finalURL := "localhost:" + tempPort - defer purge(sshResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let ssh server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptOracleAuthTest struct{} - -func (j *javascriptOracleAuthTest) Execute(filePath string) error { - if oracleResource == nil || pool == nil { - // skip test as oracle is not running - return nil - } - tempPort := oracleResource.GetPort("1521/tcp") - finalURL := "localhost:" + tempPort - defer purge(oracleResource) - - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - // let oracle server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptVncPassBrute struct{} - -func (j *javascriptVncPassBrute) Execute(filePath string) error { - if vncResource == nil || pool == nil { - // skip test as vnc is not running - return nil - } - tempPort := vncResource.GetPort("5900/tcp") - finalURL := "localhost:" + tempPort - defer purge(vncResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let ssh server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptPostgresPassBrute struct{} - -func (j *javascriptPostgresPassBrute) Execute(filePath string) error { - if postgresResource == nil || pool == nil { - // skip test as postgres is not running - return nil - } - tempPort := postgresResource.GetPort("5432/tcp") - finalURL := "localhost:" + tempPort - defer purge(postgresResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let postgres server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptMySQLConnect struct{} - -func (j *javascriptMySQLConnect) Execute(filePath string) error { - if mysqlResource == nil || pool == nil { - // skip test as mysql is not running - return nil - } - tempPort := mysqlResource.GetPort("3306/tcp") - finalURL := "localhost:" + tempPort - defer purge(mysqlResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let mysql server start - time.Sleep(5 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptMultiPortsSSH struct{} - -func (j *javascriptMultiPortsSSH) Execute(filePath string) error { - // use scanme.sh as target to ensure we match on the 2nd default port 22 - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type javascriptNoPortArgs struct{} - -func (j *javascriptNoPortArgs) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "yo.dawg", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} - -type javascriptRsyncTest struct{} - -func (j *javascriptRsyncTest) Execute(filePath string) error { - if rsyncResource == nil || pool == nil { - // skip test as rsync is not running - return nil - } - tempPort := rsyncResource.GetPort("873/tcp") - finalURL := "localhost:" + tempPort - defer purge(rsyncResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let rsync server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -type javascriptTelnetAuthTest struct{} - -func (j *javascriptTelnetAuthTest) Execute(filePath string) error { - if telnetResource == nil || pool == nil { - // skip test as telnet is not running - return nil - } - tempPort := telnetResource.GetPort("23/tcp") - finalURL := "localhost:" + tempPort - defer purge(telnetResource) - errs := []error{} - for i := 0; i < defaultRetry; i++ { - results := []string{} - var err error - _ = pool.Retry(func() error { - //let telnet server start - time.Sleep(3 * time.Second) - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug) - return nil - }) - if err != nil { - return err - } - if err := expectResultsCount(results, 1); err == nil { - return nil - } else { - errs = append(errs, err) - } - } - return multierr.Combine(errs...) -} - -// purge any given resource if it is not nil -func purge(resource *dockertest.Resource) { - if resource != nil && pool != nil { - containerName := resource.Container.Name - _ = pool.Client.StopContainer(resource.Container.ID, 0) - err := pool.Purge(resource) - if err != nil { - log.Printf("Could not purge resource: %s", err) - } - _ = pool.RemoveContainerByName(containerName) - } -} - -func init() { - // uses a sensible default on windows (tcp/http) and linux/osx (socket) - pool, err := dockertest.NewPool("") - if err != nil { - log.Printf("something went wrong with dockertest: %s", err) - return - } - - // uses pool to try to connect to Docker - err = pool.Client.Ping() - if err != nil { - log.Printf("Could not connect to Docker: %s", err) - } - - // setup a temporary redis instance - redisResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "redis", - Tag: "latest", - Cmd: []string{"redis-server", "--requirepass", "iamadmin"}, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start resource: %s", err) - return - } - // by default expire after 30 sec - if err := redisResource.Expire(30); err != nil { - log.Printf("Could not expire resource: %s", err) - } - - // setup a temporary ssh server - sshResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "lscr.io/linuxserver/openssh-server", - Tag: "latest", - Env: []string{ - "PUID=1000", - "PGID=1000", - "TZ=Etc/UTC", - "PASSWORD_ACCESS=true", - "USER_NAME=admin", - "USER_PASSWORD=admin", - }, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start resource: %s", err) - return - } - // by default expire after 30 sec - if err := sshResource.Expire(30); err != nil { - log.Printf("Could not expire resource: %s", err) - } - - // setup a temporary oracle instance - oracleResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "gvenzl/oracle-xe", - Tag: "latest", - Env: []string{ - "ORACLE_PASSWORD=mysecret", - }, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start Oracle resource: %s", err) - return - } - - // by default expire after 30 sec - if err := oracleResource.Expire(30); err != nil { - log.Printf("Could not expire Oracle resource: %s", err) - } - - // setup a temporary vnc server - vncResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "dorowu/ubuntu-desktop-lxde-vnc", - Tag: "latest", - Env: []string{ - "VNC_PASSWORD=mysecret", - }, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start resource: %s", err) - return - } - // by default expire after 30 sec - if err := vncResource.Expire(30); err != nil { - log.Printf("Could not expire resource: %s", err) - } - - // setup a temporary postgres instance - postgresResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "postgres", - Tag: "latest", - Env: []string{ - "POSTGRES_PASSWORD=postgres", - "POSTGRES_USER=postgres", - }, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start postgres resource: %s", err) - return - } - // by default expire after 30 sec - if err := postgresResource.Expire(30); err != nil { - log.Printf("Could not expire postgres resource: %s", err) - } - - // setup a temporary mysql instance - mysqlResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "mysql", - Tag: "latest", - Env: []string{ - "MYSQL_ROOT_PASSWORD=secret", - }, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start mysql resource: %s", err) - return - } - // by default expire after 30 sec - if err := mysqlResource.Expire(30); err != nil { - log.Printf("Could not expire mysql resource: %s", err) - } - - // setup a temporary rsync server - rsyncResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "alpine", - Tag: "latest", - Cmd: []string{"sh", "-c", "apk add --no-cache rsync shadow && useradd -m rsyncuser && echo 'rsyncuser:mysecret' | chpasswd && echo 'rsyncuser:MySecret123' > /etc/rsyncd.secrets && chmod 600 /etc/rsyncd.secrets && echo -e '[data]\\n path = /data\\n comment = Local Rsync Share\\n read only = false\\n auth users = rsyncuser\\n secrets file = /etc/rsyncd.secrets' > /etc/rsyncd.conf && mkdir -p /data && exec rsync --daemon --no-detach --config=/etc/rsyncd.conf"}, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start Rsync resource: %s", err) - return - } - // by default expire after 30 sec - if err := rsyncResource.Expire(30); err != nil { - log.Printf("Could not expire Rsync resource: %s", err) - } - - // setup a temporary telnet server - // username: dev - // password: mysecret - telnetResource, err = pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "alpine", - Tag: "latest", - Cmd: []string{"sh", "-c", "apk add --no-cache busybox-extras shadow && useradd -m dev && echo 'dev:mysecret' | chpasswd && exec /usr/sbin/telnetd -F -p 23 -l /bin/login"}, - Platform: "linux/amd64", - }) - if err != nil { - log.Printf("Could not start Telnet resource: %s", err) - return - } - // by default expire after 30 sec - if err := telnetResource.Expire(30); err != nil { - log.Printf("Could not expire Telnet resource: %s", err) - } -} diff --git a/cmd/integration-test/loader.go b/cmd/integration-test/loader.go deleted file mode 100644 index d6fc54644..000000000 --- a/cmd/integration-test/loader.go +++ /dev/null @@ -1,229 +0,0 @@ -package main - -import ( - "fmt" - "net/http" - "net/http/httptest" - "os" - "strings" - - "github.com/julienschmidt/httprouter" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - "github.com/projectdiscovery/utils/errkit" - permissionutil "github.com/projectdiscovery/utils/permission" -) - -var loaderTestcases = []TestCaseInfo{ - {Path: "loader/template-list.yaml", TestCase: &remoteTemplateList{}}, - {Path: "loader/workflow-list.yaml", TestCase: &remoteWorkflowList{}}, - {Path: "loader/excluded-template.yaml", TestCase: &excludedTemplate{}}, - {Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}}, - {Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}}, - {Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}}, - {Path: "loader/load-template-with-id", TestCase: &loadTemplateWithID{}}, -} - -type remoteTemplateList struct{} - -// Execute executes a test case and returns an error if occurred -func (h *remoteTemplateList) Execute(templateList string) error { - router := httprouter.New() - - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - _, _ = fmt.Fprintf(w, "This is test matcher text") - if strings.EqualFold(r.Header.Get("test"), "nuclei") { - _, _ = fmt.Fprintf(w, "This is test headers matcher text") - } - }) - - router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - file, err := os.ReadFile(templateList) - if err != nil { - w.WriteHeader(500) - } - _, err = w.Write(file) - if err != nil { - w.WriteHeader(500) - } - }) - ts := httptest.NewServer(router) - defer ts.Close() - - configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` - err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) - if err != nil { - return err - } - defer func() { - _ = os.Remove("test-config.yaml") - }() - - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml") - if err != nil { - return err - } - - return expectResultsCount(results, 2) -} - -type excludedTemplate struct{} - -// Execute executes a test case and returns an error if occurred -func (h *excludedTemplate) Execute(templateList string) error { - router := httprouter.New() - - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - _, _ = fmt.Fprintf(w, "This is test matcher text") - if strings.EqualFold(r.Header.Get("test"), "nuclei") { - _, _ = fmt.Fprintf(w, "This is test headers matcher text") - } - }) - ts := httptest.NewServer(router) - defer ts.Close() - - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-t", templateList, "-include-templates", templateList) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type remoteTemplateListNotAllowed struct{} - -// Execute executes a test case and returns an error if occurred -func (h *remoteTemplateListNotAllowed) Execute(templateList string) error { - router := httprouter.New() - - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - _, _ = fmt.Fprintf(w, "This is test matcher text") - if strings.EqualFold(r.Header.Get("test"), "nuclei") { - _, _ = fmt.Fprintf(w, "This is test headers matcher text") - } - }) - - router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - file, err := os.ReadFile(templateList) - if err != nil { - w.WriteHeader(500) - } - _, err = w.Write(file) - if err != nil { - w.WriteHeader(500) - } - }) - ts := httptest.NewServer(router) - defer ts.Close() - - _, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list") - if err == nil { - return fmt.Errorf("expected error for not allowed remote template list url") - } - - return nil - -} - -type remoteWorkflowList struct{} - -// Execute executes a test case and returns an error if occurred -func (h *remoteWorkflowList) Execute(workflowList string) error { - router := httprouter.New() - - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - _, _ = fmt.Fprintf(w, "This is test matcher text") - if strings.EqualFold(r.Header.Get("test"), "nuclei") { - _, _ = fmt.Fprintf(w, "This is test headers matcher text") - } - }) - - router.GET("/workflow_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - file, err := os.ReadFile(workflowList) - if err != nil { - w.WriteHeader(500) - } - _, err = w.Write(file) - if err != nil { - w.WriteHeader(500) - } - }) - ts := httptest.NewServer(router) - defer ts.Close() - - configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` - err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) - if err != nil { - return err - } - defer func() { - _ = os.Remove("test-config.yaml") - }() - - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml") - if err != nil { - return err - } - - return expectResultsCount(results, 3) -} - -type nonExistentTemplateList struct{} - -// Execute executes a test case and returns an error if occurred -func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error { - router := httprouter.New() - ts := httptest.NewServer(router) - defer ts.Close() - - configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` - err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) - if err != nil { - return err - } - defer func() { - _ = os.Remove("test-config.yaml") - }() - - _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml") - if err == nil { - return fmt.Errorf("expected error for nonexisting workflow url") - } - - return nil -} - -type nonExistentWorkflowList struct{} - -// Execute executes a test case and returns an error if occurred -func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error { - router := httprouter.New() - ts := httptest.NewServer(router) - defer ts.Close() - - configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]` - err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission) - if err != nil { - return err - } - defer func() { - _ = os.Remove("test-config.yaml") - }() - - _, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml") - if err == nil { - return fmt.Errorf("expected error for nonexisting workflow url") - } - - return nil -} - -type loadTemplateWithID struct{} - -func (h *loadTemplateWithID) Execute(nooop string) error { - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", "scanme.sh", "-id", "self-signed-ssl") - if err != nil { - return errkit.Wrap(err, "failed to load template with id") - } - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/network.go b/cmd/integration-test/network.go deleted file mode 100644 index 3cfe331a8..000000000 --- a/cmd/integration-test/network.go +++ /dev/null @@ -1,241 +0,0 @@ -package main - -import ( - "fmt" - "net" - "os" - "strings" - "time" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - osutils "github.com/projectdiscovery/utils/os" - "github.com/projectdiscovery/utils/reader" -) - -var networkTestcases = []TestCaseInfo{ - {Path: "protocols/network/basic.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, - {Path: "protocols/network/hex.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }}, - {Path: "protocols/network/multi-step.yaml", TestCase: &networkMultiStep{}}, - {Path: "protocols/network/self-contained.yaml", TestCase: &networkRequestSelContained{}}, - {Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}}, - {Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}}, - {Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}}, - {Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}}, - {Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}}, -} - -const defaultStaticPort = 5431 - -type networkBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *networkBasic) Execute(filePath string) error { - var routerErr error - - ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) - if err != nil { - routerErr = err - return - } - if string(data) == "PING" { - _, _ = conn.Write([]byte("PONG")) - } else { - routerErr = fmt.Errorf("invalid data received: %s", string(data)) - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err) - return err - } - if routerErr != nil { - _, _ = fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr) - return routerErr - } - - return expectResultsCount(results, 1) -} - -type networkMultiStep struct{} - -// Execute executes a test case and returns an error if occurred -func (h *networkMultiStep) Execute(filePath string) error { - var routerErr error - - ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second) - if err != nil { - routerErr = err - return - } - if string(data) == "FIRST" { - _, _ = conn.Write([]byte("PING")) - } - - data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second) - if err != nil { - routerErr = err - return - } - if string(data) == "SECOND" { - _, _ = conn.Write([]byte("PONG")) - } - _, _ = conn.Write([]byte("NUCLEI")) - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - if routerErr != nil { - return routerErr - } - - var expectedResultsSize int - if debug { - expectedResultsSize = 3 - } else { - expectedResultsSize = 1 - } - - return expectResultsCount(results, expectedResultsSize) -} - -type networkRequestSelContained struct{} - -// Execute executes a test case and returns an error if occurred -func (h *networkRequestSelContained) Execute(filePath string) error { - ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - _, _ = conn.Write([]byte("Authentication successful")) - }) - defer ts.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type networkVariables struct{} - -// Execute executes a test case and returns an error if occurred -func (h *networkVariables) Execute(filePath string) error { - var routerErr error - - ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) - if err != nil { - routerErr = err - return - } - if string(data) == "PING" { - _, _ = conn.Write([]byte("aGVsbG8=")) - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - if routerErr != nil { - return routerErr - } - - return expectResultsCount(results, 1) -} - -type networkPort struct{} - -func (n *networkPort) Execute(filePath string) error { - ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) - if err != nil { - return - } - if string(data) == "PING" { - _, _ = conn.Write([]byte("PONG")) - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - - if err := expectResultsCount(results, 1); err != nil { - return err - } - - // even though we passed port 443 in url it is ignored and port 23846 is used - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "23846", "443"), debug) - if err != nil { - return err - } - - if err := expectResultsCount(results, 1); err != nil { - return err - } - - // this is positive test case where we expect port to be overridden and 34567 to be used - ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - - data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second) - if err != nil { - return - } - if string(data) == "PING" { - _, _ = conn.Write([]byte("PONG")) - } - }) - defer ts2.Close() - - // even though we passed port 443 in url it is ignored and port 23846 is used - // instead of hardcoded port 23846 in template - results, err = testutils.RunNucleiTemplateAndGetResults(filePath, ts2.URL, debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type networkhttps struct{} - -// Execute executes a test case and returns an error if occurred -func (h *networkhttps) Execute(filePath string) error { - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) - if err != nil { - return err - } - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/ssl.go b/cmd/integration-test/ssl.go deleted file mode 100644 index c824dec68..000000000 --- a/cmd/integration-test/ssl.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "crypto/tls" - "net" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var sslTestcases = []TestCaseInfo{ - {Path: "protocols/ssl/basic.yaml", TestCase: &sslBasic{}}, - {Path: "protocols/ssl/basic-ztls.yaml", TestCase: &sslBasicZtls{}}, - {Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}}, - {Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}}, - {Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}}, - {Path: "protocols/ssl/multi-req.yaml", TestCase: &sslMultiReq{}}, -} - -type sslBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *sslBasic) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type sslBasicZtls struct{} - -// Execute executes a test case and returns an error if occurred -func (h *sslBasicZtls) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ztls") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type sslCustomCipher struct{} - -// Execute executes a test case and returns an error if occurred -func (h *sslCustomCipher) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}}, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type sslCustomVersion struct{} - -// Execute executes a test case and returns an error if occurred -func (h *sslCustomVersion) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12}, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type sslWithVars struct{} - -func (h *sslWithVars) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V", "test=asdasdas") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type sslMultiReq struct{} - -func (h *sslMultiReq) Execute(filePath string) error { - //nolint:staticcheck // SSLv3 is intentionally used for testing purposes - ts := testutils.NewTCPServer(&tls.Config{ - MinVersion: tls.VersionSSL30, - MaxVersion: tls.VersionTLS11, - }, defaultStaticPort, func(conn net.Conn) { - defer func() { - _ = conn.Close() - }() - data := make([]byte, 4) - if _, err := conn.Read(data); err != nil { - return - } - }) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V") - if err != nil { - return err - } - - return expectResultsCount(results, 2) -} diff --git a/cmd/integration-test/template-dir.go b/cmd/integration-test/template-dir.go deleted file mode 100644 index 5f26a05aa..000000000 --- a/cmd/integration-test/template-dir.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "os" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - "github.com/projectdiscovery/utils/errkit" -) - -var templatesDirTestCases = []TestCaseInfo{ - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templateDirWithTargetTest{}}, -} - -type templateDirWithTargetTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templateDirWithTargetTest) Execute(filePath string) error { - tempdir, err := os.MkdirTemp("", "nuclei-update-dir-*") - if err != nil { - return errkit.Wrap(err, "failed to create temp dir") - } - defer func() { - _ = os.RemoveAll(tempdir) - }() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "8x8exch02.8x8.com", debug, "-ud", tempdir) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/templates-dir-env.go b/cmd/integration-test/templates-dir-env.go deleted file mode 100644 index 3435c18b4..000000000 --- a/cmd/integration-test/templates-dir-env.go +++ /dev/null @@ -1,217 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - - osutils "github.com/projectdiscovery/utils/os" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" - "github.com/projectdiscovery/utils/errkit" -) - -// isNotLinux returns true if not running on Linux (used to skip tests on non-Linux OS) -var isNotLinux = func() bool { return !osutils.IsLinux() } - -var templatesDirEnvTestCases = []TestCaseInfo{ - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvBasicTest{}, DisableOn: isNotLinux}, - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvAbsolutePathTest{}, DisableOn: isNotLinux}, - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvRelativePathTest{}, DisableOn: isNotLinux}, - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvPrecedenceTest{}, DisableOn: isNotLinux}, - {Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvCustomTemplatesTest{}, DisableOn: isNotLinux}, -} - -// copyTemplateToDir copies a template file to a destination directory, preserving the directory structure -func copyTemplateToDir(templatePath, destDir string) error { - // Read the template file - templateData, err := os.ReadFile(templatePath) - if err != nil { - return errkit.Wrap(err, "failed to read template file") - } - - // Create the destination path preserving directory structure - destPath := filepath.Join(destDir, templatePath) - destDirPath := filepath.Dir(destPath) - - // Create the destination directory if it doesn't exist - if err := os.MkdirAll(destDirPath, 0755); err != nil { - return errkit.Wrap(err, "failed to create destination directory") - } - - // Write the template file - if err := os.WriteFile(destPath, templateData, 0644); err != nil { - return errkit.Wrap(err, "failed to write template file") - } - - return nil -} - -// templatesDirEnvBasicTest tests basic functionality of NUCLEI_TEMPLATES_DIR -type templatesDirEnvBasicTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templatesDirEnvBasicTest) Execute(filePath string) error { - tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*") - if err != nil { - return errkit.Wrap(err, "failed to create temp dir") - } - defer func() { - _ = os.RemoveAll(tempdir) - }() - - // Copy template to temp directory - if err := copyTemplateToDir(filePath, tempdir); err != nil { - return err - } - - // Set NUCLEI_TEMPLATES_DIR and run nuclei - envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir} - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -// templatesDirEnvAbsolutePathTest tests that absolute paths work correctly -type templatesDirEnvAbsolutePathTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templatesDirEnvAbsolutePathTest) Execute(filePath string) error { - tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-abs-*") - if err != nil { - return errkit.Wrap(err, "failed to create temp dir") - } - defer func() { - _ = os.RemoveAll(tempdir) - }() - - // Get absolute path - absTempDir, err := filepath.Abs(tempdir) - if err != nil { - return errkit.Wrap(err, "failed to get absolute path") - } - - // Copy template to temp directory - if err := copyTemplateToDir(filePath, absTempDir); err != nil { - return err - } - - // Set NUCLEI_TEMPLATES_DIR with absolute path and run nuclei - envVars := []string{"NUCLEI_TEMPLATES_DIR=" + absTempDir} - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -// templatesDirEnvRelativePathTest tests that relative paths are resolved correctly -type templatesDirEnvRelativePathTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templatesDirEnvRelativePathTest) Execute(filePath string) error { - // Create temp directory in current working directory - tempdir, err := os.MkdirTemp(".", "nuclei-templates-dir-env-rel-*") - if err != nil { - return errkit.Wrap(err, "failed to create temp dir") - } - defer func() { - _ = os.RemoveAll(tempdir) - }() - - // Get relative path (just the directory name) - relPath := filepath.Base(tempdir) - - // Copy template to temp directory - if err := copyTemplateToDir(filePath, tempdir); err != nil { - return err - } - - // Set NUCLEI_TEMPLATES_DIR with relative path and run nuclei - // Note: The implementation should convert relative paths to absolute - envVars := []string{"NUCLEI_TEMPLATES_DIR=" + relPath} - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -// templatesDirEnvPrecedenceTest tests that -ud flag takes precedence over NUCLEI_TEMPLATES_DIR -type templatesDirEnvPrecedenceTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templatesDirEnvPrecedenceTest) Execute(filePath string) error { - // Create two temp directories - envTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*") - if err != nil { - return errkit.Wrap(err, "failed to create env temp dir") - } - defer func() { - _ = os.RemoveAll(envTempDir) - }() - - flagTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-flag-*") - if err != nil { - return errkit.Wrap(err, "failed to create flag temp dir") - } - defer func() { - _ = os.RemoveAll(flagTempDir) - }() - - // Copy template to flag temp directory (this should be used due to precedence) - if err := copyTemplateToDir(filePath, flagTempDir); err != nil { - return err - } - - // Set NUCLEI_TEMPLATES_DIR to envTempDir (should be ignored due to -ud flag) - envVars := []string{"NUCLEI_TEMPLATES_DIR=" + envTempDir} - // Use -ud flag which should take precedence - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com", "-ud", flagTempDir) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -// templatesDirEnvCustomTemplatesTest tests that custom template subdirectories are correctly set -type templatesDirEnvCustomTemplatesTest struct{} - -// Execute executes a test case and returns an error if occurred -func (h *templatesDirEnvCustomTemplatesTest) Execute(filePath string) error { - tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-custom-*") - if err != nil { - return errkit.Wrap(err, "failed to create temp dir") - } - defer func() { - _ = os.RemoveAll(tempdir) - }() - - // Create custom template subdirectories structure - customDirs := []string{"github", "s3", "gitlab", "azure"} - for _, dir := range customDirs { - customDirPath := filepath.Join(tempdir, dir) - if err := os.MkdirAll(customDirPath, 0755); err != nil { - return errkit.Wrap(err, "failed to create custom template directory") - } - } - - // Copy template to temp directory - if err := copyTemplateToDir(filePath, tempdir); err != nil { - return err - } - - // Set NUCLEI_TEMPLATES_DIR and run nuclei - envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir} - results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com") - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} diff --git a/cmd/integration-test/websocket.go b/cmd/integration-test/websocket.go deleted file mode 100644 index b7d35aa71..000000000 --- a/cmd/integration-test/websocket.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "net" - "strings" - - "github.com/gobwas/ws/wsutil" - - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" -) - -var websocketTestCases = []TestCaseInfo{ - {Path: "protocols/websocket/basic.yaml", TestCase: &websocketBasic{}}, - {Path: "protocols/websocket/cswsh.yaml", TestCase: &websocketCswsh{}}, - {Path: "protocols/websocket/no-cswsh.yaml", TestCase: &websocketNoCswsh{}}, - {Path: "protocols/websocket/path.yaml", TestCase: &websocketWithPath{}}, -} - -type websocketBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *websocketBasic) Execute(filePath string) error { - connHandler := func(conn net.Conn) { - for { - msg, op, _ := wsutil.ReadClientData(conn) - if string(msg) != "hello" { - return - } - _ = wsutil.WriteServerMessage(conn, op, []byte("world")) - } - } - originValidate := func(origin string) bool { - return true - } - ts := testutils.NewWebsocketServer("", connHandler, originValidate) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type websocketCswsh struct{} - -// Execute executes a test case and returns an error if occurred -func (h *websocketCswsh) Execute(filePath string) error { - connHandler := func(conn net.Conn) { - - } - originValidate := func(origin string) bool { - return true - } - ts := testutils.NewWebsocketServer("", connHandler, originValidate) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) - if err != nil { - return err - } - - return expectResultsCount(results, 1) -} - -type websocketNoCswsh struct{} - -// Execute executes a test case and returns an error if occurred -func (h *websocketNoCswsh) Execute(filePath string) error { - connHandler := func(conn net.Conn) { - - } - originValidate := func(origin string) bool { - return origin == "https://google.com" - } - ts := testutils.NewWebsocketServer("", connHandler, originValidate) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) - if err != nil { - return err - } - - return expectResultsCount(results, 0) -} - -type websocketWithPath struct{} - -// Execute executes a test case and returns an error if occurred -func (h *websocketWithPath) Execute(filePath string) error { - connHandler := func(conn net.Conn) { - - } - originValidate := func(origin string) bool { - return origin == "https://google.com" - } - ts := testutils.NewWebsocketServer("/test", connHandler, originValidate) - defer ts.Close() - - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) - if err != nil { - return err - } - - return expectResultsCount(results, 0) -} diff --git a/cmd/tools/fuzzplayground/main.go b/cmd/tools/fuzzplayground/main.go index b65d99ed2..28ef17105 100644 --- a/cmd/tools/fuzzplayground/main.go +++ b/cmd/tools/fuzzplayground/main.go @@ -5,7 +5,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground" + "github.com/projectdiscovery/nuclei/v3/internal/fuzzplayground" ) var ( diff --git a/cmd/tools/signer/main.go b/cmd/tools/signer/main.go index 6f53b50c8..6ba61adf0 100644 --- a/cmd/tools/signer/main.go +++ b/cmd/tools/signer/main.go @@ -52,6 +52,17 @@ func main() { if err != nil { gologger.Fatal().Msgf("failed to create signer: %s", err) } + certData, err := os.ReadFile(cert) + if err != nil { + gologger.Fatal().Msgf("failed to read certificate file %s: %s", cert, err) + } + tmplVerifier, err := signer.NewTemplateSigVerifier(certData) + if err != nil { + gologger.Fatal().Msgf("failed to create verifier: %s", err) + } + if err := signer.AddSignerToDefault(tmplVerifier); err != nil { + gologger.Fatal().Msgf("failed to register verifier: %s", err) + } gologger.Info().Msgf("Template Signer: %v\n", tmplSigner.Identifier()) // read file @@ -95,6 +106,14 @@ func main() { gologger.Info().Msgf("------------------------") gologger.Info().Msgf("Signature: %s", sig2) gologger.Info().Msgf("Content Hash (SHA256): %s\n", hex.EncodeToString(hash2[:])) + + tmpl, err = templates.Parse(template, nil, execOpts) + if err != nil { + gologger.Fatal().Msgf("failed to parse signed template: %s", err) + } + } + if !tmpl.Verified { + gologger.Fatal().Msg("template signature could not be verified with the provided certificate") } gologger.Info().Msgf("✓ Template signed & verified successfully") } diff --git a/gh_retry.sh b/gh_retry.sh deleted file mode 100755 index 43542276b..000000000 --- a/gh_retry.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -# This script is used to retry failed workflows in github actions. -# It uses gh cli to fetch the failed workflows and then rerun them. -# It also checks the logs of the failed workflows to see if it is a flaky test. -# If it is a flaky test, it will rerun the failed jobs in the workflow. -# eg: -# ./gh_retry.sh -h to see the help. -# ./gh_retry.sh will run the script with default values. - -# You can also pass the following arguments: -# ./gh_retry.sh -b master -l 30 -t "30 mins ago" -w "Build Test" - -#Initialize variables to default values. -BRANCH=$(git symbolic-ref --short HEAD) -LIMIT=30 -BEFORE="30 mins ago" -WORKFLOW="Build Test" - -# You can add multiple patterns separated by | -GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed' - -#Set fonts for Help. -NORM=$(tput sgr0) -BOLD=$(tput bold) -REV=$(tput smso) - -HELP() -{ - # Display Help - echo "Script to retry failed workflows in github actions." - echo - echo "Syntax: scriptTemplate [-b]" - echo "options:" - echo "${REV}-b${NORM} Branch to check failed workflows/jobs. Default is ${BOLD}$BRANCH${NORM}." - echo "${REV}-l${NORM} Maximum number of runs to fetch. Default is ${BOLD}$LIMIT${NORM}." - echo "${REV}-t${NORM} Time to filter the failed jobs. Default is ${BOLD}$BEFORE${NORM}." - echo "${REV}-w${NORM} Workflow to filter the failed jobs. Default is ${BOLD}$WORKFLOW${NORM}." - echo -} - -while getopts :b:l:t:w:h FLAG; do - case $FLAG in - b) - BRANCH=$OPTARG - ;; - l) - LIMIT=$OPTARG - ;; - t) - BEFORE=$OPTARG - ;; - w) - WORKFLOW=$OPTARG - ;; - h) #show help - HELP - exit 0 - ;; - \?) #unrecognized option - show help - echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed." - HELP - exit 0 - ;; - esac -done -shift $((OPTIND-1)) - -function print_bold() { - echo "${BOLD}$1${NORM}" -} - -function retry_failed_jobs() { - print_bold "Checking failed workflows for branch $BRANCH before $BEFORE" - - date=$(date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE") - - workflowIds=$(gh run list --limit "$LIMIT" --json headBranch,status,name,conclusion,databaseId,updatedAt | jq -c '.[] | - select ( .headBranch==$branch ) | - select ( .name | contains($workflow) ) | - select ( .conclusion=="failure" ) | - select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" --arg workflow "$WORKFLOW" | jq .databaseId) - - # convert line separated by space to array - eval "arr=($workflowIds)" - - if [[ -z $arr ]] - then - print_bold "Could not find any failed workflows in the last $BEFORE" - exit 0 - fi - - for s in "${arr[@]}"; do - print_bold "Checking logs of failed workflow $s to see if it is a flaky test" - gh run view "$s" --log-failed | grep -E "$GREP_ERROR_PATTERN" > /dev/null - if [ $? == 0 ] ; then - print_bold "Retrying failed jobs $s" - gh run rerun "$s" --failed - sleep 10s - gh run view "$s" - fi - done -} - -retry_failed_jobs diff --git a/integration_tests/debug.sh b/integration_tests/debug.sh deleted file mode 100755 index f52127356..000000000 --- a/integration_tests/debug.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -if [ $1 = "-h" ]; then - echo "Help for ./debug.sh" - printf "\n1. To run all integration tests of 'x' protocol:\n" - printf " \$ ./debug.sh http\n\n" - printf "2. To run all integration tests of 'x' protocol that contains 'y' in template name:\n" - printf " \$ ./debug.sh http self\n\n" - printf "3. To run all integration tests of 'x' protocol that contains 'y' in template name and pass extra args to nuclei:\n" - printf " \$ ./debug.sh http self -svd -debug-req\n\n" - printf "nuclei binary is created every time script is run but integration-test binary is not" - exit 0 -fi - -# Stop execution if race condition is found -export GORACE="halt_on_error=1" - -echo "::group::Build nuclei" -rm nuclei 2>/dev/null -cd ../cmd/nuclei -go build -race . -mv nuclei ../../integration_tests/nuclei -echo -e "::endgroup::\n" -cd ../../integration_tests -cmdstring="" - -if [ -n "$1" ]; then - cmdstring+=" -protocol $1 " -fi - -if [ -n "$2" ]; then - cmdstring+=" -template $2 " -fi - -# Parse any extra args that are directly passed to nuclei -if [ -n $debugArgs ]; then - export DebugExtraArgs="${@:3}" -fi - -echo "$DebugExtraArgs" - -echo -e "::group::Run Integration Test\n" -./integration-test $cmdstring - -if [ $? -eq 0 ] -then - echo -e "::endgroup::\n" - exit 0 -else - exit 1 -fi diff --git a/integration_tests/run.sh b/integration_tests/run.sh deleted file mode 100755 index f893e1b38..000000000 --- a/integration_tests/run.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -echo "::group::Build nuclei" -rm integration-test fuzzplayground nuclei 2>/dev/null -cd ../cmd/nuclei -go build -race . -mv nuclei ../../integration_tests/nuclei -echo "::endgroup::" - -echo "::group::Build nuclei integration-test" -cd ../integration-test -go build -mv integration-test ../../integration_tests/integration-test -cd ../../integration_tests -echo "::endgroup::" - -echo "::group::Installing nuclei templates" -./nuclei -update-templates -echo "::endgroup::" - -./integration-test -if [ $? -eq 0 ] -then - exit 0 -else - exit 1 -fi diff --git a/pkg/testutils/fuzzplayground/db.go b/internal/fuzzplayground/db.go similarity index 100% rename from pkg/testutils/fuzzplayground/db.go rename to internal/fuzzplayground/db.go diff --git a/pkg/testutils/fuzzplayground/server.go b/internal/fuzzplayground/server.go similarity index 100% rename from pkg/testutils/fuzzplayground/server.go rename to internal/fuzzplayground/server.go diff --git a/pkg/testutils/fuzzplayground/sqli_test.go b/internal/fuzzplayground/sqli_test.go similarity index 100% rename from pkg/testutils/fuzzplayground/sqli_test.go rename to internal/fuzzplayground/sqli_test.go diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 60b3df9d5..5a7ee713f 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -2,6 +2,7 @@ package runner import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -12,14 +13,14 @@ import ( func TestCreateReportingOptions(t *testing.T) { var options types.Options - options.ReportingConfig = "../../integration_tests/test-issue-tracker-config1.yaml" + options.ReportingConfig = filepath.Join("testdata", "test-issue-tracker-config1.yaml") resultOptions, err := createReportingOptions(&options) require.Nil(t, err) require.Equal(t, resultOptions.AllowList.Severities, severity.Severities{severity.High, severity.Critical}) require.Equal(t, resultOptions.DenyList.Severities, severity.Severities{severity.Low}) - options.ReportingConfig = "../../integration_tests/test-issue-tracker-config2.yaml" + options.ReportingConfig = filepath.Join("testdata", "test-issue-tracker-config2.yaml") resultOptions2, err := createReportingOptions(&options) require.Nil(t, err) require.Equal(t, resultOptions2.AllowList.Severities, resultOptions.AllowList.Severities) diff --git a/internal/runner/testdata/test-issue-tracker-config1.yaml b/internal/runner/testdata/test-issue-tracker-config1.yaml new file mode 100644 index 000000000..9ef6c18ba --- /dev/null +++ b/internal/runner/testdata/test-issue-tracker-config1.yaml @@ -0,0 +1,68 @@ +allow-list: + severity: high, critical +deny-list: + severity: low + +# GitHub contains configuration options for GitHub issue tracker +github: + # base-url is the optional self-hosted GitHub application url + base-url: https://localhost:8443/github + # username is the username of the GitHub user + username: test-username + # owner is the owner name of the repository for issues + owner: test-owner + # token is the token for GitHub account + token: test-token + # project-name is the name of the repository + project-name: test-project + # issue-label is the label of the created issue type + issue-label: bug + +# GitLab contains configuration options for gitlab issue tracker +gitlab: + # base-url is the optional self-hosted GitLab application url + base-url: https://localhost:8443/gitlab + # username is the username of the GitLab user + username: test-username + # token is the token for GitLab account + token: test-token + # project-name is the name/id of the project(repository) + project-name: "1234" + # issue-label is the label of the created issue type + issue-label: bug + +# Jira contains configuration options for Jira issue tracker +jira: + # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used + cloud: true + # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created + update-existing: false + # URL is the jira application url + url: https://localhost/jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira + account-id: test-account-id + # email is the email of the user for Jira instance + email: test@test.com + # token is the token for Jira instance or password in case of on-prem Jira + token: test-token + # project-name is the name of the project. + project-name: test-project-name + # issue-type is the name of the created issue type + issue-type: bug + +# elasticsearch contains configuration options for elasticsearch exporter +elasticsearch: + # IP for elasticsearch instance + ip: 127.0.0.1 + # Port is the port of elasticsearch instance + port: 9200 + # IndexName is the name of the elasticsearch index + index-name: nuclei + # SSL enables ssl for elasticsearch connection + ssl: false + # SSLVerification disables SSL verification for elasticsearch + ssl-verification: false + # Username for the elasticsearch instance + username: test + # Password is the password for elasticsearch instance + password: test \ No newline at end of file diff --git a/integration_tests/test-issue-tracker-config2.yaml b/internal/runner/testdata/test-issue-tracker-config2.yaml similarity index 100% rename from integration_tests/test-issue-tracker-config2.yaml rename to internal/runner/testdata/test-issue-tracker-config2.yaml diff --git a/internal/tests/functional/functional_test.go b/internal/tests/functional/functional_test.go new file mode 100644 index 000000000..f80324533 --- /dev/null +++ b/internal/tests/functional/functional_test.go @@ -0,0 +1,499 @@ +//go:build functional +// +build functional + +package functional_test + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + + "github.com/google/shlex" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +type functionalHarness struct { + repoRoot string + testcasesPath string + configDir string + releaseBinary string + currentBinary string + releaseRunner *testutils.Runner + currentRunner *testutils.Runner + debug bool + ci bool + runner *testutils.Runner +} + +var suite functionalHarness + +func TestMain(m *testing.M) { + os.Exit(runTestMain(m)) +} + +func runTestMain(m *testing.M) int { + wd, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to determine working directory: %v\n", err) + return 1 + } + + repoRoot := filepath.Clean(filepath.Join(wd, "../../..")) + configDir, err := os.MkdirTemp("", "nuclei-functional-config-*") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create functional config dir: %v\n", err) + return 1 + } + suite = functionalHarness{ + repoRoot: repoRoot, + testcasesPath: filepath.Join(repoRoot, "internal", "tests", "functional", "testdata", "testcases.txt"), + configDir: configDir, + debug: isDebugMode(), + ci: isCI(), + runner: testutils.NewRunner( + testutils.WithWorkingDir(repoRoot), + testutils.WithBaseEnv("NUCLEI_CONFIG_DIR="+configDir), + ), + } + defer func() { + _ = os.RemoveAll(configDir) + }() + + if suite.ci { + releaseBinary, err := resolveFunctionalBinary(os.Getenv("RELEASE_BINARY"), "nuclei") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to find release nuclei binary on PATH: %v\n", err) + return 1 + } + currentBinary, err := resolveCurrentFunctionalBinary(repoRoot) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + suite.releaseBinary = releaseBinary + suite.currentBinary = currentBinary + suite.releaseRunner = suite.runner.Clone(testutils.WithBinaryPath(releaseBinary)) + suite.currentRunner = suite.runner.Clone(testutils.WithBinaryPath(currentBinary)) + if err := prepareFunctionalEnvironment(currentBinary, repoRoot, configDir); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + } + + return m.Run() +} + +func TestFunctionalComparison(t *testing.T) { + if !suite.ci { + t.Skip("functional suite requires CI=true; use make functional-dev for local release-vs-dev comparisons") + } + + testcases, err := loadFunctionalTestcases(suite.testcasesPath) + if err != nil { + t.Fatalf("failed to load functional testcases: %v", err) + } + semaphore := make(chan struct{}, functionalParallelism()) + + for index, testcase := range testcases { + testcase := testcase + name := functionalTestName(index, testcase.raw) + t.Run(name, func(t *testing.T) { + t.Parallel() + semaphore <- struct{}{} + defer func() { <-semaphore }() + if err := runFunctionalCase(testcase.args, suite.debug); err != nil { + t.Fatal(err) + } + }) + } +} + +type functionalTestcase struct { + raw string + args []string +} + +func loadFunctionalTestcases(path string) ([]functionalTestcase, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + + var testcases []functionalTestcase + seen := make(map[string]struct{}) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if _, ok := seen[line]; ok { + continue + } + args := splitFunctionalArgs(line) + if len(args) == 0 { + continue + } + seen[line] = struct{}{} + testcases = append(testcases, functionalTestcase{raw: line, args: args}) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return testcases, nil +} + +func splitFunctionalArgs(testcase string) []string { + parts, err := shlex.Split(testcase) + if err != nil { + return nil + } + if len(parts) <= 1 { + return nil + } + args := parts[1:] + restoreRawFlagValues(args, "-tc", extractRawFlagValues(testcase, "-tc")) + return args +} + +func extractRawFlagValues(testcase, flagName string) []string { + values := make([]string, 0, 1) + for searchStart := 0; searchStart < len(testcase); { + flagIndex := findFlagIndex(testcase[searchStart:], flagName) + if flagIndex == -1 { + break + } + + flagStart := searchStart + flagIndex + valueStart := flagStart + len(flagName) + for valueStart < len(testcase) && isFunctionalArgSpace(testcase[valueStart]) { + valueStart++ + } + if valueStart >= len(testcase) { + break + } + + valueEnd := scanFlagValueEnd(testcase, valueStart) + values = append(values, trimMatchingQuotes(strings.TrimSpace(testcase[valueStart:valueEnd]))) + searchStart = valueEnd + } + return values +} + +func findFlagIndex(input, flagName string) int { + for searchIndex := 0; searchIndex < len(input); { + candidate := strings.Index(input[searchIndex:], flagName) + if candidate == -1 { + return -1 + } + candidate += searchIndex + if functionalFlagBoundary(input, candidate, len(flagName)) { + return candidate + } + searchIndex = candidate + len(flagName) + } + return -1 +} + +func functionalFlagBoundary(input string, start, length int) bool { + if start > 0 && !isFunctionalArgSpace(input[start-1]) { + return false + } + end := start + length + return end == len(input) || isFunctionalArgSpace(input[end]) +} + +func scanFlagValueEnd(input string, start int) int { + quote := byte(0) + for index := start; index < len(input); index++ { + char := input[index] + if quote != 0 { + if char == quote { + quote = 0 + } + continue + } + switch char { + case '\'', '"': + quote = char + case ' ', '\t', '\n', '\r': + next := index + for next < len(input) && isFunctionalArgSpace(input[next]) { + next++ + } + if next < len(input) && input[next] == '-' && next+1 < len(input) && isFunctionalFlagStart(input[next+1]) { + return index + } + } + } + return len(input) +} + +func isFunctionalArgSpace(char byte) bool { + switch char { + case ' ', '\t', '\n', '\r': + return true + default: + return false + } +} + +func isFunctionalFlagStart(char byte) bool { + return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') +} + +func restoreRawFlagValues(args []string, flagName string, rawValues []string) { + if len(rawValues) == 0 { + return + } + + rawIndex := 0 + for index := 0; index < len(args)-1 && rawIndex < len(rawValues); index++ { + if args[index] != flagName { + continue + } + args[index+1] = rawValues[rawIndex] + rawIndex++ + } +} + +func trimMatchingQuotes(value string) string { + if len(value) < 2 { + return value + } + if (value[0] == '\'' && value[len(value)-1] == '\'') || (value[0] == '"' && value[len(value)-1] == '"') { + return value[1 : len(value)-1] + } + return value +} + +func TestSplitFunctionalArgs(t *testing.T) { + t.Run("quoted values", func(t *testing.T) { + args := splitFunctionalArgs(`{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical`) + expected := []string{"-tags", "cve,exposure", "-author", "geeknik,pdteam", "-severity", "high,critical"} + if strings.Join(args, "\x00") != strings.Join(expected, "\x00") { + t.Fatalf("unexpected args: got=%q want=%q", args, expected) + } + }) + + t.Run("tc expression preserves quotes", func(t *testing.T) { + args := splitFunctionalArgs(`{{binary}} -tags cve -author geeknik,pdteam -tc severity=='high'`) + expected := []string{"-tags", "cve", "-author", "geeknik,pdteam", "-tc", "severity=='high'"} + if strings.Join(args, "\x00") != strings.Join(expected, "\x00") { + t.Fatalf("unexpected args: got=%q want=%q", args, expected) + } + }) + + t.Run("tc expression before next flag", func(t *testing.T) { + args := splitFunctionalArgs(`{{binary}} -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml`) + expected := []string{"-tc", "contains(tags,'cve')", "-exclude-templates", "http/cves/2020/CVE-2020-9757.yaml"} + if strings.Join(args, "\x00") != strings.Join(expected, "\x00") { + t.Fatalf("unexpected args: got=%q want=%q", args, expected) + } + }) +} + +func functionalTestName(index int, raw string) string { + name := raw + name = strings.TrimPrefix(name, "{{binary}}") + name = strings.TrimSpace(name) + name = strings.ReplaceAll(name, " ", "_") + name = strings.ReplaceAll(name, "/", "-") + if len(name) > 80 { + name = name[:80] + } + if name == "" { + name = "binary" + } + return fmt.Sprintf("%03d_%s", index+1, name) +} + +func functionalParallelism() int { + for _, envName := range []string{"FUNCTIONAL_PARALLELISM", "PARALLELISM"} { + if value := strings.TrimSpace(os.Getenv(envName)); value != "" { + if parsed, err := strconv.Atoi(value); err == nil && parsed > 0 { + return parsed + } + } + } + processBudget := max(min(runtime.GOMAXPROCS(0), 4), 1) + return max((processBudget+1)/2, 1) +} + +func runFunctionalCase(args []string, debug bool) error { + if suite.releaseRunner == nil || suite.currentRunner == nil { + return fmt.Errorf("functional runners are not initialized") + } + + var ( + releaseOutput string + currentOutput string + releaseErr error + currentErr error + ) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + releaseOutput, releaseErr = suite.releaseRunner.LoadedTemplates("", debug, args) + }() + go func() { + defer wg.Done() + currentOutput, currentErr = suite.currentRunner.LoadedTemplates("", debug, args) + }() + wg.Wait() + + if releaseErr != nil { + return fmt.Errorf("could not run release nuclei test: %w", releaseErr) + } + if currentErr != nil { + return fmt.Errorf("could not run current nuclei test: %w", currentErr) + } + if releaseOutput != currentOutput { + return fmt.Errorf("release loaded %s templates but current loaded %s", releaseOutput, currentOutput) + } + return nil +} + +func buildCurrentBinary(repoRoot, binaryPath string) error { + if err := os.MkdirAll(filepath.Dir(binaryPath), 0755); err != nil { + return fmt.Errorf("failed to create bin directory: %w", err) + } + cmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/nuclei") + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to build current nuclei binary: %w\n%s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func resolveCurrentFunctionalBinary(repoRoot string) (string, error) { + configuredBinary := strings.TrimSpace(os.Getenv("DEV_BINARY")) + if configuredBinary != "" { + if resolved, ok := resolveExistingBinaryPath(configuredBinary); ok { + return resolved, nil + } + } + currentBinary := filepath.Join(repoRoot, "bin", binaryName()) + if err := buildCurrentBinary(repoRoot, currentBinary); err != nil { + return "", err + } + return currentBinary, nil +} + +func resolveFunctionalBinary(configuredPath, executableName string) (string, error) { + if trimmed := strings.TrimSpace(configuredPath); trimmed != "" { + if resolved, ok := resolveExistingBinaryPath(trimmed); ok { + return resolved, nil + } + } + for _, candidate := range candidateExecutableNames(executableName) { + resolved, err := exec.LookPath(candidate) + if err == nil { + return resolved, nil + } + } + return "", fmt.Errorf("could not locate %s on PATH", executableName) +} + +func resolveExistingBinaryPath(candidate string) (string, bool) { + trimmed := strings.Trim(strings.TrimSpace(candidate), `"'`) + if trimmed == "" { + return "", false + } + + for _, pathCandidate := range candidateExecutableNames(trimmed) { + absolutePath, err := filepath.Abs(pathCandidate) + if err != nil { + continue + } + info, statErr := os.Stat(absolutePath) + if statErr == nil && !info.IsDir() { + return absolutePath, true + } + } + + for _, pathCandidate := range candidateExecutableNames(trimmed) { + resolved, err := exec.LookPath(pathCandidate) + if err == nil { + return resolved, true + } + } + + return "", false +} + +func candidateExecutableNames(name string) []string { + trimmed := strings.Trim(strings.TrimSpace(name), `"'`) + if trimmed == "" { + return nil + } + + candidates := []string{trimmed} + if runtime.GOOS == "windows" && filepath.Ext(trimmed) == "" { + candidates = append(candidates, trimmed+".exe") + } + return candidates +} + +func prepareFunctionalEnvironment(binaryPath, repoRoot, configDir string) error { + configRoot := filepath.Join(configDir, "nuclei") + if err := os.MkdirAll(configRoot, 0755); err != nil { + return fmt.Errorf("failed to create functional nuclei config root: %w", err) + } + ignoreFile := filepath.Join(configRoot, ".nuclei-ignore") + if err := os.WriteFile(ignoreFile, nil, 0644); err != nil { + return fmt.Errorf("failed to create functional ignore file: %w", err) + } + for _, args := range [][]string{{"-update-templates"}, {"-validate"}} { + cmd := exec.Command(binaryPath, args...) + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), "NUCLEI_CONFIG_DIR="+configDir) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("functional setup command %v failed: %w\n%s", args, err, strings.TrimSpace(string(output))) + } + } + return nil +} + +func binaryName() string { + if runtime.GOOS == "windows" { + return "nuclei.exe" + } + return "nuclei" +} + +func isCI() bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv("CI"))) + return value == "true" || value == "1" +} + +func isDebugMode() bool { + for _, envVar := range []string{"DEBUG", "ACTIONS_STEP_DEBUG", "ACTIONS_RUNNER_DEBUG", "RUNNER_DEBUG"} { + if envTruthy(envVar) { + return true + } + } + return false +} + +func envTruthy(name string) bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(name))) + switch value { + case "true", "1", "yes", "on", "enabled": + return true + default: + return false + } +} diff --git a/cmd/functional-test/targets-1000.txt b/internal/tests/functional/testdata/targets-1000.txt similarity index 100% rename from cmd/functional-test/targets-1000.txt rename to internal/tests/functional/testdata/targets-1000.txt diff --git a/cmd/functional-test/targets-150.txt b/internal/tests/functional/testdata/targets-150.txt similarity index 100% rename from cmd/functional-test/targets-150.txt rename to internal/tests/functional/testdata/targets-150.txt diff --git a/cmd/functional-test/targets-250.txt b/internal/tests/functional/testdata/targets-250.txt similarity index 100% rename from cmd/functional-test/targets-250.txt rename to internal/tests/functional/testdata/targets-250.txt diff --git a/cmd/functional-test/targets.txt b/internal/tests/functional/testdata/targets.txt similarity index 100% rename from cmd/functional-test/targets.txt rename to internal/tests/functional/testdata/targets.txt diff --git a/cmd/functional-test/testcases.txt b/internal/tests/functional/testdata/testcases.txt similarity index 86% rename from cmd/functional-test/testcases.txt rename to internal/tests/functional/testdata/testcases.txt index 31796a229..c867cc93c 100644 --- a/cmd/functional-test/testcases.txt +++ b/internal/tests/functional/testdata/testcases.txt @@ -2,73 +2,60 @@ {{binary}} # Template tags filter -{{binary}} -tags cve -ntv 8.8.8,8.8.9 -{{binary}} -tags cve -{{binary}} -tags cve,exposure -{{binary}} -tags cve,exposure -tags token -{{binary}} -tags cve,exposure -tags token,logs -{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -t http/cves/ +{{binary}} -t http/cves/ -t http/exposures/ +{{binary}} -t http/cves/ -t http/exposures/ -tags config +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/ +{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml +{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf {{binary}} -tags 'cve','exposure' -tags 'token','logs' -{{binary}} -tags cve -severity high -{{binary}} -tags cve,exposure -severity high,critical -{{binary}} -tags cve,exposure -severity high,critical,medium -{{binary}} -tags cve -author geeknik -{{binary}} -tags cve -author geeknik,pdteam -{{binary}} -tags cve -author geeknik -severity high -{{binary}} -tags cve -{{binary}} -tags cve,exposure -{{binary}} -tags cve,exposure -tags token -{{binary}} -tags cve,exposure -tags token,logs -{{binary}} -tags "cve","exposure" -tags "token","logs" -{{binary}} -tags 'cve','exposure' -tags 'token','logs' -{{binary}} -tags cve -severity high -{{binary}} -tags cve,exposure -severity high,critical -{{binary}} -tags cve,exposure -severity high,critical,medium -{{binary}} -tags cve -author geeknik -{{binary}} -tags cve -author geeknik,pdteam -{{binary}} -tags cve -author geeknik -severity high -{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical {{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical +{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -tags cve +{{binary}} -tags cve -author geeknik +{{binary}} -tags cve -author geeknik -severity high +{{binary}} -tags cve -author geeknik,pdteam {{binary}} -tags cve -etags ssrf -{{binary}} -tags cve,exposure -etags ssrf,config -{{binary}} -tags cve,exposure -etags ssrf,config -severity high -{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik -{{binary}} -tags cve,dos,fuzz -{{binary}} -tags cve -include-tags dos,fuzz {{binary}} -tags cve -exclude-tags cve2020 {{binary}} -tags cve -exclude-templates http/cves/2020/ {{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml {{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml -exclude-templates http/cves/2021/ -{{binary}} -t http/cves/ -{{binary}} -t http/cves/ -t http/exposures/ -{{binary}} -t http/cves/ -t http/exposures/ -tags config -{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/ -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml -{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml +{{binary}} -tags cve -include-tags dos,fuzz +{{binary}} -tags cve -ntv 8.8.8,8.8.9 +{{binary}} -tags cve -severity high +{{binary}} -tags cve,dos,fuzz +{{binary}} -tags cve,exposure +{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical +{{binary}} -tags cve,exposure -etags ssrf,config +{{binary}} -tags cve,exposure -etags ssrf,config -severity high +{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik +{{binary}} -tags cve,exposure -severity high,critical +{{binary}} -tags cve,exposure -severity high,critical,medium +{{binary}} -tags cve,exposure -tags token +{{binary}} -tags cve,exposure -tags token,logs # Advanced Filtering +{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml {{binary}} -tags cve -author geeknik,pdteam -tc severity=='high' {{binary}} -tc contains(authors,'pdteam') -{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml -{{binary}} -tc protocol=='dns' -{{binary}} -tc contains(http_method,'GET') -{{binary}} -tc len(body)>0 -{{binary}} -tc contains(matcher_type,'word') -{{binary}} -tc contains(extractor_type,'regex') {{binary}} -tc contains(description,'wordpress') +{{binary}} -tc contains(extractor_type,'regex') +{{binary}} -tc contains(http_method,'GET') +{{binary}} -tc contains(matcher_type,'word') +{{binary}} -tc len(body)>0 +{{binary}} -tc protocol=='dns' # Workflow Filters {{binary}} -w workflows {{binary}} -w workflows -author geeknik,pdteam -{{binary}} -w workflows -severity high,critical {{binary}} -w workflows -author geeknik,pdteam -severity high,critical +{{binary}} -w workflows -severity high,critical # Input Types -# http protocol # host {{binary}} -id tech-detect -u scanme.sh # host:port @@ -91,5 +78,5 @@ {{binary}} -id tls-version -u scanme.sh:22 # Options -# Tls Impersonate +# TLS Impersonate {{binary}} -id tech-detect -tlsi -u https://scanme.sh \ No newline at end of file diff --git a/cmd/integration-test/code.go b/internal/tests/integration/code_test.go similarity index 67% rename from cmd/integration-test/code.go rename to internal/tests/integration/code_test.go index c2c591084..d80c66ab3 100644 --- a/cmd/integration-test/code.go +++ b/internal/tests/integration/code_test.go @@ -1,21 +1,35 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "errors" - "log" "os" - "path/filepath" + "os/exec" + "sync" osutils "github.com/projectdiscovery/utils/os" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/templates" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" ) var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" } +var signedCodeTemplates sync.Once +var signedCodeTemplatesErr error -var codeTestCases = []TestCaseInfo{ +func hasAnyExecutable(names ...string) bool { + for _, name := range names { + if _, err := exec.LookPath(name); err == nil { + return true + } + } + return false +} + +var codeTestCases = []integrationCase{ {Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled}, @@ -24,9 +38,9 @@ var codeTestCases = []TestCaseInfo{ {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }}, {Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled}, - {Path: "protocols/code/sh-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }}, - {Path: "protocols/code/py-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }}, - {Path: "protocols/code/pwsh-echo.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return isCodeDisabled() }}, + {Path: "protocols/code/sh-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() || !hasAnyExecutable("docker", "podman") }, Serial: true}, + {Path: "protocols/code/py-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() || !hasAnyExecutable("docker", "podman") }, Serial: true}, + {Path: "protocols/code/pwsh-echo.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return isCodeDisabled() || !hasAnyExecutable("pwsh", "powershell", "powershell.exe") }}, } const ( @@ -36,49 +50,62 @@ const ( var testcertpath = "" -func init() { +func ensureSignedCodeTemplates() error { if isCodeDisabled() { - // skip executing code protocol in CI on windows - return + return nil } - // allow local file access to load content of file references in template - // in order to sign them for testing purposes - templates.TemplateSignerLFA() - - tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile) - if err != nil { - panic(err) - } - - testcertpath, _ = filepath.Abs(testCertFile) - - for _, v := range codeTestCases { - templatePath := v.Path - testCase := v.TestCase - - if v.DisableOn != nil && v.DisableOn() { - // skip ps1 test case on non-windows platforms - continue - } - - templatePath, err := filepath.Abs(templatePath) + signedCodeTemplates.Do(func() { + previousWD, err := os.Getwd() if err != nil { - panic(err) + signedCodeTemplatesErr = err + return + } + if err := os.Chdir(suite.fixturesDir); err != nil { + signedCodeTemplatesErr = err + return + } + defer func() { + if chdirErr := os.Chdir(previousWD); chdirErr != nil && signedCodeTemplatesErr == nil { + signedCodeTemplatesErr = chdirErr + } + }() + + templates.TemplateSignerLFA() + + certPath := fixturePath(testCertFile) + keyPath := fixturePath(testKeyFile) + testcertpath = certPath + + tsigner, err := signer.NewTemplateSignerFromFiles(certPath, keyPath) + if err != nil { + signedCodeTemplatesErr = err + return } - // skip - // - unsigned test cases - if _, ok := testCase.(*unsignedCode); ok { - continue + templatesToSign := []string{ + "workflow/code-template-1.yaml", + "workflow/code-template-2.yaml", } - if _, ok := testCase.(*codePyNoSig); ok { - continue + for _, v := range codeTestCases { + if v.DisableOn != nil && v.DisableOn() { + continue + } + if _, ok := v.TestCase.(*unsignedCode); ok { + continue + } + if _, ok := v.TestCase.(*codePyNoSig); ok { + continue + } + templatesToSign = append(templatesToSign, v.Path) } - if err := templates.SignTemplate(tsigner, templatePath); err != nil { - log.Fatalf("Could not sign template %v got: %s\n", templatePath, err) + for _, templatePath := range templatesToSign { + if err := templates.SignTemplate(tsigner, fixturePath(templatePath)); err != nil { + signedCodeTemplatesErr = err + return + } } - } - + }) + return signedCodeTemplatesErr } func getEnvValues() []string { diff --git a/internal/tests/integration/config_and_templates_test.go b/internal/tests/integration/config_and_templates_test.go new file mode 100644 index 000000000..4e2282f5b --- /dev/null +++ b/internal/tests/integration/config_and_templates_test.go @@ -0,0 +1,142 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "slices" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func newTemplateDirTarget(t *testing.T) string { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("This is test matcher text")) + })) + t.Cleanup(server.Close) + return server.URL +} + +func TestTemplateDir(t *testing.T) { + t.Run("WithTarget", func(t *testing.T) { + tempDir := t.TempDir() + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/http/get.yaml", newTemplateDirTarget(t), suite.debug, "-ud", tempDir) + if err != nil { + t.Fatalf("template dir override request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} + +func TestTemplatesDirEnv(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("NUCLEI_TEMPLATES_DIR integration cases are only supported on Linux") + } + target := newTemplateDirTarget(t) + + t.Run("Basic", func(t *testing.T) { + tempDir := t.TempDir() + copyFixtureToDir(t, "protocols/http/get.yaml", tempDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target) + if err != nil { + t.Fatalf("templates dir env basic request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("AbsolutePath", func(t *testing.T) { + tempDir := t.TempDir() + copyFixtureToDir(t, "protocols/http/get.yaml", tempDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target) + if err != nil { + t.Fatalf("templates dir env absolute path request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("RelativePath", func(t *testing.T) { + tempDir, err := os.MkdirTemp(suite.fixturesDir, "nuclei-templates-dir-env-rel-*") + if err != nil { + t.Fatalf("failed to create relative templates dir: %v", err) + } + defer func() { _ = os.RemoveAll(tempDir) }() + + copyFixtureToDir(t, "protocols/http/get.yaml", tempDir) + relPath := filepath.Base(tempDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + relPath}, "-t", "protocols/http/get.yaml", "-u", target) + if err != nil { + t.Fatalf("templates dir env relative path request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("Precedence", func(t *testing.T) { + envTempDir := t.TempDir() + flagTempDir := t.TempDir() + copyFixtureToDir(t, "protocols/http/get.yaml", flagTempDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + envTempDir}, "-t", "protocols/http/get.yaml", "-u", target, "-ud", flagTempDir) + if err != nil { + t.Fatalf("templates dir env precedence request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CustomSubdirs", func(t *testing.T) { + tempDir := t.TempDir() + for _, dir := range []string{"github", "s3", "gitlab", "azure"} { + if err := os.MkdirAll(filepath.Join(tempDir, dir), 0755); err != nil { + t.Fatalf("failed to create custom templates subdir %s: %v", dir, err) + } + } + copyFixtureToDir(t, "protocols/http/get.yaml", tempDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target) + if err != nil { + t.Fatalf("templates dir env custom subdirs request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} + +func TestCustomConfigDir(t *testing.T) { + t.Run("ConfigDirIsolated", func(t *testing.T) { + customTempDir := t.TempDir() + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDir}, "-t", "protocols/http/get.yaml", "-u", newTemplateDirTarget(t)) + if err != nil { + t.Fatalf("custom config dir request failed: %v", err) + } + if len(results) != 0 { + files, err := os.ReadDir(customTempDir) + if err != nil { + t.Fatalf("failed to inspect custom config dir: %v", err) + } + var fileNames []string + for _, file := range files { + fileNames = append(fileNames, file.Name()) + } + for _, requiredFile := range []string{".templates-config.json", "config.yaml", "reporting-config.yaml"} { + if !slices.Contains(fileNames, requiredFile) { + t.Fatalf("missing required config file %q in custom config dir: %v", requiredFile, fileNames) + } + } + } + }) +} diff --git a/internal/tests/integration/dns_test.go b/internal/tests/integration/dns_test.go new file mode 100644 index 000000000..ca5915ace --- /dev/null +++ b/internal/tests/integration/dns_test.go @@ -0,0 +1,140 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func TestDNS(t *testing.T) { + t.Run("A", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/a.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns A request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("AAAA", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/aaaa.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns AAAA request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CNAME", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/cname.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns CNAME request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("SRV", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/srv.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns SRV request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("NS", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/ns.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns NS request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("TXT", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/txt.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns TXT request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("PTR", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/ptr.yaml", "1.1.1.1", suite.debug) + if err != nil { + t.Fatalf("dns PTR request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CAA", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/caa.yaml", "pki.goog", suite.debug) + if err != nil { + t.Fatalf("dns CAA request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("TLSA", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/tlsa.yaml", "scanme.sh", suite.debug) + if err != nil { + t.Fatalf("dns TLSA request failed: %v", err) + } + if err := expectResultsCount(results, 0); err != nil { + t.Fatal(err) + } + }) + + t.Run("Variables", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/variables.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns variables request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("Payload", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/payload.yaml", "google.com", suite.debug) + if err != nil { + t.Fatalf("dns payload request failed: %v", err) + } + if err := expectResultsCount(results, 3); err != nil { + t.Fatal(err) + } + + results, err = testutils.RunNucleiTemplateAndGetResults("protocols/dns/payload.yaml", "google.com", suite.debug, "-var", "subdomain_wordlist=subdomains.txt") + if err != nil { + t.Fatalf("dns payload request with CLI override failed: %v", err) + } + if err := expectResultsCount(results, 4); err != nil { + t.Fatal(err) + } + }) + + t.Run("DSLMatcherVariable", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/dsl-matcher-variable.yaml", "one.one.one.one", suite.debug) + if err != nil { + t.Fatalf("dns DSL matcher variable request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/integration-test/dsl.go b/internal/tests/integration/dsl_test.go similarity index 88% rename from cmd/integration-test/dsl.go rename to internal/tests/integration/dsl_test.go index c311b8292..e8a1aa90f 100644 --- a/cmd/integration-test/dsl.go +++ b/internal/tests/integration/dsl_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" @@ -6,10 +9,10 @@ import ( "net/http/httptest" "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) -var dslTestcases = []TestCaseInfo{ +var dslTestcases = []integrationCase{ {Path: "dsl/hide-version-warning.yaml", TestCase: &dslVersionWarning{}}, {Path: "dsl/show-version-warning.yaml", TestCase: &dslShowVersionWarning{}}, } diff --git a/cmd/integration-test/exporters.go b/internal/tests/integration/exporters_test.go similarity index 96% rename from cmd/integration-test/exporters.go rename to internal/tests/integration/exporters_test.go index 7890734f4..ac12d1d3e 100644 --- a/cmd/integration-test/exporters.go +++ b/internal/tests/integration/exporters_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "context" @@ -21,7 +24,7 @@ const ( dbImage = "mongo:8" ) -var exportersTestCases = []TestCaseInfo{ +var exportersTestCases = []integrationCase{ {Path: "exporters/mongo", TestCase: &mongoExporter{}, DisableOn: func() bool { return osutil.IsWindows() || osutil.IsOSX() }}, diff --git a/internal/tests/integration/file_test.go b/internal/tests/integration/file_test.go new file mode 100644 index 000000000..b30db86b4 --- /dev/null +++ b/internal/tests/integration/file_test.go @@ -0,0 +1,55 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "path/filepath" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func TestFile(t *testing.T) { + target := filepath.ToSlash(filepath.Join("protocols", "file", "data")) + "/" + + t.Run("MatcherWithOr", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-or.yaml", target, suite.debug, "-file") + if err != nil { + t.Fatalf("file OR matcher request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("MatcherWithAnd", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-and.yaml", target, suite.debug, "-file") + if err != nil { + t.Fatalf("file AND matcher request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("MatcherWithNestedAnd", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-nested-and.yaml", target, suite.debug, "-file") + if err != nil { + t.Fatalf("file nested AND matcher request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("Extract", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/extract.yaml", target, suite.debug, "-file") + if err != nil { + t.Fatalf("file extractor request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} diff --git a/internal/tests/integration/flow_test.go b/internal/tests/integration/flow_test.go new file mode 100644 index 000000000..3b50a96af --- /dev/null +++ b/internal/tests/integration/flow_test.go @@ -0,0 +1,98 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func TestFlow(t *testing.T) { + t.Run("ConditionalFlow", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("flow/conditional-flow.yaml", "cloud.projectdiscovery.io", suite.debug) + if err != nil { + t.Fatalf("conditional flow request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("ConditionalFlowNegative", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("flow/conditional-flow-negative.yaml", "scanme.sh", suite.debug) + if err != nil { + t.Fatalf("conditional negative flow request failed: %v", err) + } + if err := expectResultsCount(results, 0); err != nil { + t.Fatal(err) + } + }) + + t.Run("IterateValuesFlow", func(t *testing.T) { + router := httprouter.New() + testEmails := []string{"secrets@scanme.sh", "superadmin@scanme.sh"} + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, testEmails) + }) + router.GET("/user/"+getBase64(testEmails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + router.GET("/user/"+getBase64(testEmails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) + }) + server := httptest.NewServer(router) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("flow/iterate-values-flow.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("iterate values flow request failed: %v", err) + } + if err := expectResultsCount(results, 2); err != nil { + t.Fatal(err) + } + }) + + t.Run("IterateOneValueFlow", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("flow/iterate-one-value-flow.yaml", "https://scanme.sh", suite.debug) + if err != nil { + t.Fatalf("iterate one value flow request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("DNSNSProbe", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("flow/dns-ns-probe.yaml", "oast.fun", suite.debug) + if err != nil { + t.Fatalf("dns ns probe flow request failed: %v", err) + } + if err := expectResultsCount(results, 2); err != nil { + t.Fatal(err) + } + }) + + t.Run("HideMatcher", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("flow/flow-hide-matcher.yaml", "scanme.sh", suite.debug) + if err != nil { + t.Fatalf("hide matcher flow request failed: %v", err) + } + if err := expectResultsCount(results, 0); err != nil { + t.Fatal(err) + } + }) +} + +func getBase64(input string) string { + return base64.StdEncoding.EncodeToString([]byte(input)) +} diff --git a/cmd/integration-test/fuzz.go b/internal/tests/integration/fuzz_test.go similarity index 97% rename from cmd/integration-test/fuzz.go rename to internal/tests/integration/fuzz_test.go index c2c4bee3a..87fbe2189 100644 --- a/cmd/integration-test/fuzz.go +++ b/internal/tests/integration/fuzz_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" @@ -7,8 +10,8 @@ import ( "net/url" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" ) @@ -16,7 +19,7 @@ const ( targetFile = "fuzz/testData/ginandjuice.proxify.yaml" ) -var fuzzingTestCases = []TestCaseInfo{ +var fuzzingTestCases = []integrationCase{ {Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}}, {Path: "fuzz/fuzz-multi-mode.yaml", TestCase: &fuzzMultipleMode{}}, {Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}}, diff --git a/cmd/integration-test/generic.go b/internal/tests/integration/generic_test.go similarity index 96% rename from cmd/integration-test/generic.go rename to internal/tests/integration/generic_test.go index f5edfc946..ac1fa2ad6 100644 --- a/cmd/integration-test/generic.go +++ b/internal/tests/integration/generic_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "crypto/tls" @@ -10,11 +13,11 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" permissionutil "github.com/projectdiscovery/utils/permission" ) -var genericTestcases = []TestCaseInfo{ +var genericTestcases = []integrationCase{ {Path: "generic/auth/certificate/http-get.yaml", TestCase: &clientCertificate{}}, } diff --git a/cmd/integration-test/headless.go b/internal/tests/integration/headless_test.go similarity index 97% rename from cmd/integration-test/headless.go rename to internal/tests/integration/headless_test.go index abc2a0368..23a2ed437 100644 --- a/cmd/integration-test/headless.go +++ b/internal/tests/integration/headless_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" @@ -8,10 +11,10 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) -var headlessTestcases = []TestCaseInfo{ +var headlessTestcases = []integrationCase{ {Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessBasic{}}, {Path: "protocols/headless/headless-waitevent.yaml", TestCase: &headlessBasic{}}, {Path: "protocols/headless/headless-dsl.yaml", TestCase: &headlessBasic{}}, diff --git a/cmd/integration-test/http.go b/internal/tests/integration/http_test.go similarity index 96% rename from cmd/integration-test/http.go rename to internal/tests/integration/http_test.go index 52b4ad62e..e692e6c87 100644 --- a/cmd/integration-test/http.go +++ b/internal/tests/integration/http_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "errors" @@ -7,6 +10,7 @@ import ( "net/http/httptest" "net/http/httputil" "os" + "path/filepath" "reflect" "regexp" "strconv" @@ -17,7 +21,7 @@ import ( "github.com/julienschmidt/httprouter" "gopkg.in/yaml.v2" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/errkit" @@ -27,7 +31,7 @@ import ( unitutils "github.com/projectdiscovery/utils/unit" ) -var httpTestcases = []TestCaseInfo{ +var httpTestcases = []integrationCase{ // TODO: excluded due to parsing errors with console // "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, {Path: "protocols/http/get-headers.yaml", TestCase: &httpGetHeaders{}}, @@ -797,7 +801,7 @@ func (h *httpRawUnsafePath) Execute(filepath string) error { // testing unsafe paths using router feedback is not possible cause they are `unsafe urls` // hence it is done by parsing and matching paths from nuclei output with `-debug-req` flag // read template files - bin, err := os.ReadFile(filepath) + bin, err := os.ReadFile(fixturePath(filepath)) if err != nil { return err } @@ -845,7 +849,7 @@ type httpPaths struct{} func (h *httpPaths) Execute(filepath string) error { // covers testcases similar to httpRawUnsafePath but when `unsafe:false` - bin, err := os.ReadFile(filepath) + bin, err := os.ReadFile(fixturePath(filepath)) if err != nil { return err } @@ -995,18 +999,17 @@ func (h *httpRequestSelfContained) Execute(filePath string) error { router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { _, _ = w.Write([]byte("This is self-contained response")) }) - server := &http.Server{ - Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), - Handler: router, - } - go func() { - _ = server.ListenAndServe() - }() - defer func() { - _ = server.Close() - }() + server := httptest.NewServer(router) + defer server.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc") + templatePath, err := tempFixturePath(filePath, map[string]string{ + "http://127.0.0.1:5431": server.URL, + "127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"), + }) + if err != nil { + return err + } + results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-esc") if err != nil { return err } @@ -1033,18 +1036,17 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error { } _, _ = w.Write([]byte("This is self-contained response")) }) - server := &http.Server{ - Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), - Handler: router, - } - go func() { - _ = server.ListenAndServe() - }() - defer func() { - _ = server.Close() - }() + server := httptest.NewServer(router) + defer server.Close() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc") + templatePath, err := tempFixturePath(filePath, map[string]string{ + "http://127.0.0.1:5431": server.URL, + "127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"), + }) + if err != nil { + return err + } + results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-esc") if err != nil { return err } @@ -1068,16 +1070,8 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error { gotReqToEndpoints = append(gotReqToEndpoints, "/two") _, _ = w.Write([]byte("This is self-contained response")) }) - server := &http.Server{ - Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), - Handler: router, - } - go func() { - _ = server.ListenAndServe() - }() - defer func() { - _ = server.Close() - }() + server := httptest.NewServer(router) + defer server.Close() // create temp file FileLoc, err := os.CreateTemp("", "self-contained-payload-*.txt") @@ -1091,7 +1085,14 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error { _ = FileLoc.Close() }() - results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc.Name(), "-esc") + templatePath, err := tempFixturePath(filePath, map[string]string{ + "http://127.0.0.1:5431": server.URL, + "127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"), + }) + if err != nil { + return err + } + results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-V", "test="+FileLoc.Name(), "-esc") if err != nil { return err } @@ -1106,6 +1107,26 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error { return nil } +func tempFixturePath(relativePath string, replacements map[string]string) (string, error) { + data, err := os.ReadFile(fixturePath(relativePath)) + if err != nil { + return "", fmt.Errorf("failed to read fixture %s: %w", relativePath, err) + } + content := string(data) + for oldValue, newValue := range replacements { + content = strings.ReplaceAll(content, oldValue, newValue) + } + tempDir, err := os.MkdirTemp(suite.tempDir, "http-self-contained-*") + if err != nil { + return "", fmt.Errorf("failed to create temp fixture dir: %w", err) + } + tempPath := filepath.Join(tempDir, filepath.Base(relativePath)) + if err := os.WriteFile(tempPath, []byte(content), 0600); err != nil { + return "", fmt.Errorf("failed to write temp fixture %s: %w", relativePath, err) + } + return tempPath, nil +} + type httpGetCaseInsensitive struct{} // Execute executes a test case and returns an error if occurred diff --git a/internal/tests/integration/integration_test.go b/internal/tests/integration/integration_test.go new file mode 100644 index 000000000..746924893 --- /dev/null +++ b/internal/tests/integration/integration_test.go @@ -0,0 +1,279 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/fuzzplayground" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +type integrationHarness struct { + repoRoot string + fixturesDir string + binaryPath string + debug bool + tempDir string + runner *testutils.Runner +} + +var suite integrationHarness +var debug bool + +func TestMain(m *testing.M) { + wd, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to determine working directory: %v\n", err) + os.Exit(1) + } + + repoRoot := filepath.Clean(filepath.Join(wd, "../../..")) + tempDir, err := os.MkdirTemp("", "nuclei-integration-*") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create integration temp dir: %v\n", err) + os.Exit(1) + } + + binaryName := "nuclei" + if runtime.GOOS == "windows" { + binaryName += ".exe" + } + binaryPath := filepath.Join(tempDir, binaryName) + if err := buildNucleiBinary(repoRoot, binaryPath); err != nil { + fmt.Fprintln(os.Stderr, err) + _ = os.RemoveAll(tempDir) + os.Exit(1) + } + + workingFixturesDir := filepath.Join(tempDir, "fixtures") + if err := copyDir(filepath.Join(repoRoot, "internal", "tests", "integration", "testdata"), workingFixturesDir); err != nil { + fmt.Fprintf(os.Stderr, "failed to stage integration fixtures: %v\n", err) + _ = os.RemoveAll(tempDir) + os.Exit(1) + } + + previousRunner := testutils.DefaultRunner() + runner := testutils.NewRunner( + testutils.WithBinaryPath(binaryPath), + testutils.WithWorkingDir(workingFixturesDir), + testutils.WithExtraArgs(integrationExtraArgs...), + ) + testutils.SetDefaultRunner(runner) + + suite = integrationHarness{ + repoRoot: repoRoot, + fixturesDir: workingFixturesDir, + binaryPath: binaryPath, + debug: isDebugMode(), + tempDir: tempDir, + runner: runner, + } + debug = suite.debug + + fuzzServer := fuzzplayground.GetPlaygroundServer() + defer func() { + fuzzplayground.Cleanup() + _ = fuzzServer.Close() + }() + go func() { + if err := fuzzServer.Start("localhost:8082"); err != nil && !strings.Contains(err.Error(), "Server closed") { + fmt.Fprintf(os.Stderr, "failed to start fuzz playground: %v\n", err) + } + }() + + exitCode := m.Run() + testutils.SetDefaultRunner(previousRunner) + _ = os.RemoveAll(tempDir) + os.Exit(exitCode) +} + +func buildNucleiBinary(repoRoot, binaryPath string) error { + cmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/nuclei") + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to build nuclei binary: %w\n%s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func isDebugMode() bool { + for _, envVar := range []string{"DEBUG", "ACTIONS_STEP_DEBUG", "ACTIONS_RUNNER_DEBUG", "RUNNER_DEBUG"} { + if envTruthy(envVar) { + return true + } + } + return false +} + +func isCI() bool { + return envTruthy("CI") +} + +func envTruthy(name string) bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv(name))) + switch value { + case "true", "1", "yes": + return true + default: + return false + } +} + +func fixturePath(relativePath string) string { + if filepath.IsAbs(relativePath) { + return relativePath + } + return filepath.Join(suite.fixturesDir, filepath.FromSlash(relativePath)) +} + +func expectResultsCount(results []string, expectedNumbers ...int) error { + results = filterLines(results) + actualCount := len(results) + for _, expected := range expectedNumbers { + if actualCount == expected { + return nil + } + } + return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected)\nresults:\n\t%s", actualCount, expectedNumbers, strings.Join(results, "\n\t")) +} + +type capturedError struct { + mu sync.Mutex + err error +} + +func (c *capturedError) Set(err error) { + if err == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.err = err + } +} + +func (c *capturedError) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func tempFixtureCopy(t *testing.T, relativePath string, replacements map[string]string) string { + t.Helper() + + data, err := os.ReadFile(fixturePath(relativePath)) + if err != nil { + t.Fatalf("failed to read fixture %s: %v", relativePath, err) + } + content := string(data) + if len(replacements) > 0 { + replacementPairs := make([]string, 0, len(replacements)*2) + for oldValue, newValue := range replacements { + replacementPairs = append(replacementPairs, oldValue, newValue) + } + content = strings.NewReplacer(replacementPairs...).Replace(content) + } + tempPath := filepath.Join(t.TempDir(), filepath.Base(relativePath)) + if err := os.WriteFile(tempPath, []byte(content), 0600); err != nil { + t.Fatalf("failed to write temp fixture %s: %v", relativePath, err) + } + return tempPath +} + +func copyFixtureToDir(t *testing.T, relativePath, destDir string) string { + t.Helper() + + srcPath := fixturePath(relativePath) + destPath := filepath.Join(destDir, filepath.FromSlash(relativePath)) + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + t.Fatalf("failed to create fixture destination for %s: %v", relativePath, err) + } + if err := copyFile(srcPath, destPath, 0644); err != nil { + t.Fatalf("failed to copy fixture %s: %v", relativePath, err) + } + return destPath +} + +func copyDir(srcDir, destDir string) error { + return filepath.WalkDir(srcDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + if relPath == "." { + return os.MkdirAll(destDir, 0755) + } + targetPath := filepath.Join(destDir, relPath) + if entry.IsDir() { + return os.MkdirAll(targetPath, 0755) + } + info, err := entry.Info() + if err != nil { + return err + } + return copyFile(path, targetPath, info.Mode()) + }) +} + +func copyFile(srcPath, destPath string, mode fs.FileMode) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer func() { _ = srcFile.Close() }() + + destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + defer func() { _ = destFile.Close() }() + + _, err = io.Copy(destFile, srcFile) + return err +} + +func filterLines(results []string) []string { + results = filterHeadlessLogs(results) + results = filterUnsignedTemplatesWarnings(results) + return results +} + +func filterHeadlessLogs(results []string) []string { + filtered := make([]string, 0, len(results)) + for _, result := range results { + if strings.Contains(result, "[launcher.Browser]") { + continue + } + filtered = append(filtered, result) + } + return filtered +} + +func filterUnsignedTemplatesWarnings(results []string) []string { + filtered := make([]string, 0, len(results)) + unsignedTemplatesRegex := regexp.MustCompile(`Loading \d+ unsigned templates for scan\. Use with caution\.`) + for _, result := range results { + if unsignedTemplatesRegex.MatchString(result) { + continue + } + filtered = append(filtered, result) + } + return filtered +} diff --git a/cmd/integration-test/interactsh.go b/internal/tests/integration/interactsh_test.go similarity index 87% rename from cmd/integration-test/interactsh.go rename to internal/tests/integration/interactsh_test.go index 03df594cf..480c3a7e6 100644 --- a/cmd/integration-test/interactsh.go +++ b/internal/tests/integration/interactsh_test.go @@ -1,9 +1,12 @@ -package main +//go:build integration +// +build integration + +package integration_test import osutils "github.com/projectdiscovery/utils/os" // All Interactsh related testcases -var interactshTestCases = []TestCaseInfo{ +var interactshTestCases = []integrationCase{ {Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }}, {Path: "protocols/http/interactsh-with-payloads.yaml", TestCase: &httpInteractshWithPayloadsRequest{}, DisableOn: func() bool { return true }}, {Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return true }}, // disable this test for now diff --git a/internal/tests/integration/javascript_test.go b/internal/tests/integration/javascript_test.go new file mode 100644 index 000000000..c132571c4 --- /dev/null +++ b/internal/tests/integration/javascript_test.go @@ -0,0 +1,372 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "database/sql" + "fmt" + "net" + "time" + + "github.com/go-pg/pg/v10" + _ "github.com/go-sql-driver/mysql" + "github.com/ory/dockertest/v3" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" + osutils "github.com/projectdiscovery/utils/os" + "github.com/projectdiscovery/utils/reader" + "go.uber.org/multierr" +) + +func javascriptDockerDisabled() bool { + return !osutils.IsLinux() || !hasAnyExecutable("docker", "podman") +} + +var jsTestcases = []integrationCase{ + {Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}}, + {Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}}, + {Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/mysql-connect.yaml", TestCase: &javascriptMySQLConnect{}, DisableOn: javascriptDockerDisabled, Serial: true}, + {Path: "protocols/javascript/multi-ports.yaml", TestCase: &javascriptMultiPortsSSH{}}, + {Path: "protocols/javascript/no-port-args.yaml", TestCase: &javascriptNoPortArgs{}}, + {Path: "protocols/javascript/telnet-auth-test.yaml", TestCase: &javascriptTelnetAuthTest{}, DisableOn: javascriptDockerDisabled, Serial: true}, +} + +var ( + defaultRetry = 3 +) + +const ( + javascriptContainerTTLSeconds = 300 + javascriptDatabaseReadyTimeout = 3 * time.Minute + javascriptServiceReadyTimeout = 45 * time.Second + javascriptRetryDelay = 500 * time.Millisecond + javascriptDialTimeout = 500 * time.Millisecond +) + +type javascriptDockerReadyCheck func(address string) error + +type javascriptDockerSpec struct { + options *dockertest.RunOptions + port string + readyTimeout time.Duration + settleDelay time.Duration + readyCheck javascriptDockerReadyCheck +} + +func newJavascriptDockerSpec(port string, options *dockertest.RunOptions, readyTimeout time.Duration, settleDelay time.Duration, readyCheck javascriptDockerReadyCheck) javascriptDockerSpec { + if options == nil { + options = &dockertest.RunOptions{} + } + if options.Platform == "" { + options.Platform = "linux/amd64" + } + if len(options.ExposedPorts) == 0 { + options.ExposedPorts = []string{port} + } + if readyTimeout <= 0 { + readyTimeout = javascriptServiceReadyTimeout + } + return javascriptDockerSpec{ + options: options, + port: port, + readyTimeout: readyTimeout, + settleDelay: settleDelay, + readyCheck: readyCheck, + } +} + +type javascriptNetHttps struct{} + +func (j *javascriptNetHttps) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + +type javascriptRedisPassBrute struct{} + +func (j *javascriptRedisPassBrute) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("6379/tcp", &dockertest.RunOptions{ + Repository: "redis", + Tag: "latest", + Cmd: []string{"redis-server", "--requirepass", "iamadmin"}, + }, javascriptServiceReadyTimeout, 0, nil)) +} + +type javascriptSSHServerFingerprint struct{} + +func (j *javascriptSSHServerFingerprint) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("22/tcp", &dockertest.RunOptions{ + Repository: "alpine", + Tag: "latest", + Cmd: []string{ + "sh", + "-c", + "apk add --no-cache openssh && ssh-keygen -A && mkdir -p /run/sshd && exec /usr/sbin/sshd -D -e", + }, + }, javascriptServiceReadyTimeout, 2*time.Second, nil)) +} + +type javascriptVncPassBrute struct{} + +func (j *javascriptVncPassBrute) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("5900/tcp", &dockertest.RunOptions{ + Repository: "dorowu/ubuntu-desktop-lxde-vnc", + Tag: "latest", + Env: []string{ + "VNC_PASSWORD=mysecret", + }, + }, javascriptServiceReadyTimeout, 20*time.Second, nil)) +} + +type javascriptPostgresPassBrute struct{} + +func (j *javascriptPostgresPassBrute) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("5432/tcp", &dockertest.RunOptions{ + Repository: "postgres", + Tag: "13", + Env: []string{ + "POSTGRES_INITDB_ARGS=--auth-host=md5 --auth-local=trust", + "POSTGRES_PASSWORD=postgres", + "POSTGRES_USER=postgres", + }, + }, javascriptDatabaseReadyTimeout, 0, postgresReadyCheck("postgres", "postgres")), 1, 2) +} + +type javascriptMySQLConnect struct{} + +func (j *javascriptMySQLConnect) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("3306/tcp", &dockertest.RunOptions{ + Repository: "mysql", + Tag: "5.7", + Env: []string{ + "MYSQL_ROOT_PASSWORD=secret", + }, + }, javascriptDatabaseReadyTimeout, 0, mysqlReadyCheck("root", "secret"))) +} + +type javascriptMultiPortsSSH struct{} + +func (j *javascriptMultiPortsSSH) Execute(filePath string) error { + // use scanme.sh as target to ensure we match on the 2nd default port 22 + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + +type javascriptNoPortArgs struct{} + +func (j *javascriptNoPortArgs) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "yo.dawg", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + +type javascriptRsyncTest struct{} + +func (j *javascriptRsyncTest) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("873/tcp", &dockertest.RunOptions{ + Repository: "alpine", + Tag: "latest", + Cmd: []string{"sh", "-c", "apk add --no-cache rsync shadow && useradd -m rsyncuser && echo 'rsyncuser:mysecret' | chpasswd && echo 'rsyncuser:MySecret123' > /etc/rsyncd.secrets && chmod 600 /etc/rsyncd.secrets && echo -e '[data]\\n path = /data\\n comment = Local Rsync Share\\n read only = false\\n auth users = rsyncuser\\n secrets file = /etc/rsyncd.secrets' > /etc/rsyncd.conf && mkdir -p /data && exec rsync --daemon --no-detach --config=/etc/rsyncd.conf"}, + }, javascriptServiceReadyTimeout, 0, nil)) +} + +type javascriptTelnetAuthTest struct{} + +func (j *javascriptTelnetAuthTest) Execute(filePath string) error { + return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("23/tcp", &dockertest.RunOptions{ + Repository: "alpine", + Tag: "latest", + Cmd: []string{"sh", "-c", "apk add --no-cache busybox-extras shadow && useradd -m dev && echo 'dev:mysecret' | chpasswd && exec /usr/sbin/telnetd -F -p 23 -l /bin/login"}, + }, javascriptServiceReadyTimeout, 0, nil)) +} + +type networkMultiStep struct{} + +func (j *networkMultiStep) Execute(filePath string) error { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "FIRST" { + _, _ = conn.Write([]byte("PING")) + } + + data, err = reader.ConnReadNWithTimeout(conn, 6, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "SECOND" { + _, _ = conn.Write([]byte("PONG")) + } + _, _ = conn.Write([]byte("NUCLEI")) + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, server.URL, debug) + if err != nil { + return err + } + if err := errState.Err(); err != nil { + return err + } + if debug { + return expectResultsCount(results, 3) + } + return expectResultsCount(results, 1) +} + +func runJavascriptDockerCase(filePath string, spec javascriptDockerSpec, expectedNumbers ...int) error { + if len(expectedNumbers) == 0 { + expectedNumbers = []int{1} + } + + if spec.options == nil { + return fmt.Errorf("missing docker options for %s", filePath) + } + + pool, err := dockertest.NewPool("") + if err != nil { + return fmt.Errorf("could not create docker pool: %w", err) + } + if err := pool.Client.Ping(); err != nil { + return fmt.Errorf("could not connect to Docker: %w", err) + } + + resource, err := pool.RunWithOptions(spec.options) + if err != nil { + return fmt.Errorf("could not start resource for %s: %w", filePath, err) + } + defer purge(pool, resource) + + if err := resource.Expire(javascriptContainerTTLSeconds); err != nil { + return fmt.Errorf("could not expire resource for %s: %w", filePath, err) + } + + mappedPort := resource.GetPort(spec.port) + if mappedPort == "" { + return fmt.Errorf("missing mapped port for %s", spec.port) + } + + targetAddress := net.JoinHostPort("127.0.0.1", mappedPort) + if err := waitForTCPService(targetAddress, spec.readyTimeout); err != nil { + return err + } + if spec.readyCheck != nil { + if err := waitForServiceCheck(targetAddress, spec.readyTimeout, spec.readyCheck); err != nil { + return err + } + } + if spec.settleDelay > 0 { + time.Sleep(spec.settleDelay) + } + + errS := make([]error, 0, defaultRetry) + for attempt := 1; attempt <= defaultRetry; attempt++ { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, targetAddress, debug) + if err == nil { + if countErr := expectResultsCount(results, expectedNumbers...); countErr == nil { + return nil + } else { + err = countErr + } + } + errS = append(errS, err) + if attempt < defaultRetry { + time.Sleep(javascriptRetryDelay) + } + } + + return multierr.Combine(errS...) +} + +func waitForTCPService(address string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", address, javascriptDialTimeout) + if err == nil { + _ = conn.Close() + return nil + } + lastErr = err + time.Sleep(javascriptRetryDelay) + } + if lastErr == nil { + lastErr = fmt.Errorf("timed out waiting for %s", address) + } + return fmt.Errorf("service %s did not become ready: %w", address, lastErr) +} + +func waitForServiceCheck(address string, timeout time.Duration, check javascriptDockerReadyCheck) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if err := check(address); err == nil { + return nil + } else { + lastErr = err + } + time.Sleep(javascriptRetryDelay) + } + if lastErr == nil { + lastErr = fmt.Errorf("timed out waiting for %s", address) + } + return fmt.Errorf("service %s did not become fully ready: %w", address, lastErr) +} + +func mysqlReadyCheck(username, password string) javascriptDockerReadyCheck { + return func(address string) error { + dsn := fmt.Sprintf("%s:%s@tcp(%s)/information_schema?timeout=5s&readTimeout=5s&writeTimeout=5s", username, password, address) + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(0) + return db.Ping() + } +} + +func postgresReadyCheck(username, password string) javascriptDockerReadyCheck { + return func(address string) error { + db := pg.Connect(&pg.Options{ + Addr: address, + User: username, + Password: password, + Database: "postgres", + }).WithTimeout(5 * time.Second) + defer func() { _ = db.Close() }() + _, err := db.Exec("select 1") + return err + } +} + +func purge(pool *dockertest.Pool, resource *dockertest.Resource) { + if resource == nil || pool == nil { + return + } + containerName := resource.Container.Name + _ = pool.Client.StopContainer(resource.Container.ID, 0) + _ = pool.Purge(resource) + _ = pool.RemoveContainerByName(containerName) +} diff --git a/cmd/integration-test/library.go b/internal/tests/integration/library_test.go similarity index 91% rename from cmd/integration-test/library.go rename to internal/tests/integration/library_test.go index 8b61c3ffc..a2d01533b 100644 --- a/cmd/integration-test/library.go +++ b/internal/tests/integration/library_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "context" @@ -6,8 +9,7 @@ import ( "log" "net/http" "net/http/httptest" - "os" - "path" + "path/filepath" "strings" "time" @@ -16,6 +18,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" @@ -30,12 +33,11 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/ratelimit" ) -var libraryTestcases = []TestCaseInfo{ +var libraryTestcases = []integrationCase{ {Path: "library/test.yaml", TestCase: &goIntegrationTest{}}, {Path: "library/test.json", TestCase: &goIntegrationTest{}}, } @@ -85,7 +87,11 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) defer protocolstate.Close(defaultOpts.ExecutionId) - defaultOpts.Templates = goflags.StringSlice{templatePath} + actualTemplatePath := templatePath + if !filepath.IsAbs(actualTemplatePath) { + actualTemplatePath = fixturePath(actualTemplatePath) + } + defaultOpts.Templates = goflags.StringSlice{actualTemplatePath} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags outputWriter := testutils.NewMockOutputWriter(defaultOpts.OmitTemplate) @@ -101,8 +107,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) } defer interactClient.Close() - home, _ := os.UserHomeDir() - catalog := disk.NewCatalog(path.Join(home, "nuclei-templates")) + catalog := disk.NewCatalog(suite.fixturesDir) ratelimiter := ratelimit.New(context.Background(), 150, time.Second) defer ratelimiter.Stop() diff --git a/internal/tests/integration/loader_test.go b/internal/tests/integration/loader_test.go new file mode 100644 index 000000000..3a6737880 --- /dev/null +++ b/internal/tests/integration/loader_test.go @@ -0,0 +1,166 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" + "github.com/projectdiscovery/utils/errkit" + permissionutil "github.com/projectdiscovery/utils/permission" +) + +func TestLoader(t *testing.T) { + t.Run("RemoteTemplateList", func(t *testing.T) { + server := newLoaderServer(t, "/template_list", fixturePath("loader/template-list.yaml")) + + configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String()) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-template-url", server.URL+"/template_list", + "-config", configPath, + ) + if err != nil { + t.Fatalf("expected remote template list to succeed: %v", err) + } + if err := expectResultsCount(results, 2); err != nil { + t.Fatal(err) + } + }) + + t.Run("ExcludedTemplate", func(t *testing.T) { + server := newLoaderServer(t, "", "") + + templatePath := fixturePath("loader/excluded-template.yaml") + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-t", templatePath, + "-include-templates", templatePath, + ) + if err != nil { + t.Fatalf("expected excluded template case to succeed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("RemoteTemplateListNotAllowed", func(t *testing.T) { + server := newLoaderServer(t, "/template_list", fixturePath("loader/template-list.yaml")) + + _, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-template-url", server.URL+"/template_list", + ) + if err == nil { + t.Fatal("expected error for remote template list without allow-list config") + } + }) + + t.Run("RemoteWorkflowList", func(t *testing.T) { + server := newLoaderServer(t, "/workflow_list", fixturePath("loader/workflow-list.yaml")) + + configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String()) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-workflow-url", server.URL+"/workflow_list", + "-config", configPath, + ) + if err != nil { + t.Fatalf("expected remote workflow list to succeed: %v", err) + } + if err := expectResultsCount(results, 3); err != nil { + t.Fatal(err) + } + }) + + t.Run("NonExistentTemplateList", func(t *testing.T) { + server := newLoaderServer(t, "", "") + + configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String()) + _, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-template-url", server.URL+"/404", + "-config", configPath, + ) + if err == nil { + t.Fatal("expected error for non-existing remote template list") + } + }) + + t.Run("NonExistentWorkflowList", func(t *testing.T) { + server := newLoaderServer(t, "", "") + + configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String()) + _, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil, + "-target", server.URL, + "-workflow-url", server.URL+"/404", + "-config", configPath, + ) + if err == nil { + t.Fatal("expected error for non-existing remote workflow list") + } + }) + + t.Run("LoadTemplateWithID", func(t *testing.T) { + server := newLoaderServer(t, "", "") + + templateDir := t.TempDir() + copyFixtureToDir(t, "library/test.yaml", templateDir) + results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + templateDir}, + "-target", server.URL, + "-id", "go-integration-test", + ) + if err != nil { + t.Fatalf("failed to load template with id: %v", errkit.Wrap(err, "load template with id")) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} + +func newLoaderServer(t *testing.T, listPath, listFixture string) *httptest.Server { + t.Helper() + + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = fmt.Fprint(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + _, _ = fmt.Fprint(w, "This is test headers matcher text") + } + }) + if listPath != "" { + router.GET(listPath, func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(listFixture) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = w.Write(file) + }) + } + + server := httptest.NewServer(router) + t.Cleanup(server.Close) + return server +} + +func writeAllowedRemoteConfig(t *testing.T, allowedAddress string) string { + t.Helper() + + configPath := filepath.Join(t.TempDir(), "test-config.yaml") + configData := []byte(`remote-template-domain: [ "` + allowedAddress + `" ]`) + if err := os.WriteFile(configPath, configData, permissionutil.ConfigFilePermission); err != nil { + t.Fatalf("failed to write remote template allow-list config: %v", err) + } + return configPath +} diff --git a/cmd/integration-test/matcher-status.go b/internal/tests/integration/matcher-status_test.go similarity index 96% rename from cmd/integration-test/matcher-status.go rename to internal/tests/integration/matcher-status_test.go index 70fac1bd9..f65c78a94 100644 --- a/cmd/integration-test/matcher-status.go +++ b/internal/tests/integration/matcher-status_test.go @@ -1,4 +1,7 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" @@ -7,12 +10,12 @@ import ( "strings" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" ) -var matcherStatusTestcases = []TestCaseInfo{ +var matcherStatusTestcases = []integrationCase{ {Path: "protocols/http/get.yaml", TestCase: &httpNoAccess{}}, {Path: "protocols/network/net-https.yaml", TestCase: &networkNoAccess{}}, {Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessNoAccess{}}, diff --git a/cmd/integration-test/multi.go b/internal/tests/integration/multi_test.go similarity index 78% rename from cmd/integration-test/multi.go rename to internal/tests/integration/multi_test.go index 7915d96cb..0d671ab95 100644 --- a/cmd/integration-test/multi.go +++ b/internal/tests/integration/multi_test.go @@ -1,10 +1,13 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) -var multiProtoTestcases = []TestCaseInfo{ +var multiProtoTestcases = []integrationCase{ {Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}}, {Path: "protocols/multi/evaluate-variables.yaml", TestCase: &multiProtoDynamicExtractor{}}, {Path: "protocols/multi/exported-response-vars.yaml", TestCase: &multiProtoDynamicExtractor{}}, diff --git a/internal/tests/integration/network_test.go b/internal/tests/integration/network_test.go new file mode 100644 index 000000000..dc6375504 --- /dev/null +++ b/internal/tests/integration/network_test.go @@ -0,0 +1,262 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" + "github.com/projectdiscovery/utils/reader" +) + +func TestNetwork(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + return + } + errState.Set(fmt.Errorf("invalid data received: %s", string(data))) + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/basic.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("basic network request failed: %v", err) + } + if err := errState.Err(); err != nil { + t.Fatal(err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("Hex", func(t *testing.T) { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + return + } + errState.Set(fmt.Errorf("invalid data received: %s", string(data))) + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/hex.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("hex network request failed: %v", err) + } + if err := errState.Err(); err != nil { + t.Fatal(err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("MultiStep", func(t *testing.T) { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "FIRST" { + _, _ = conn.Write([]byte("PING")) + } + + data, err = reader.ConnReadNWithTimeout(conn, 6, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "SECOND" { + _, _ = conn.Write([]byte("PONG")) + } + _, _ = conn.Write([]byte("NUCLEI")) + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/multi-step.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("multi-step network request failed: %v", err) + } + if err := errState.Err(); err != nil { + t.Fatal(err) + } + expectedCount := 1 + if suite.debug { + expectedCount = 3 + } + if err := expectResultsCount(results, expectedCount); err != nil { + t.Fatal(err) + } + }) + + t.Run("SelfContained", func(t *testing.T) { + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + _, _ = conn.Write([]byte("Authentication successful")) + }) + defer server.Close() + + templatePath := tempFixtureCopy(t, "protocols/network/self-contained.yaml", map[string]string{ + "127.0.0.1:5431": server.URL, + }) + results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", suite.debug, "-esc") + if err != nil { + t.Fatalf("self-contained network request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("Variables", func(t *testing.T) { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "PING" { + _, _ = conn.Write([]byte("aGVsbG8=")) + } + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/variables.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("variables network request failed: %v", err) + } + if err := errState.Err(); err != nil { + t.Fatal(err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("SameAddress", func(t *testing.T) { + errState := &capturedError{} + server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err != nil { + errState.Set(err) + return + } + if string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + return + } + errState.Set(fmt.Errorf("invalid data received: %s", string(data))) + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/same-address.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("same-address network request failed: %v", err) + } + if err := errState.Err(); err != nil { + t.Fatal(err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("NetworkPort", func(t *testing.T) { + server := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err == nil && string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + } + }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("network-port template failed with template port: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + + results, err = testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", strings.ReplaceAll(server.URL, "23846", "443"), suite.debug) + if err != nil { + t.Fatalf("network-port template failed with overridden input port: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + + serverOverride := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + + data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second) + if err == nil && string(data) == "PING" { + _, _ = conn.Write([]byte("PONG")) + } + }) + defer serverOverride.Close() + + results, err = testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", serverOverride.URL, suite.debug) + if err != nil { + t.Fatalf("network-port template failed with runtime override port: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("NetHTTPS", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/net-https.yaml", "scanme.sh", suite.debug) + if err != nil { + t.Fatalf("network https request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("NetHTTPSTimeout", func(t *testing.T) { + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/net-https-timeout.yaml", "scanme.sh", suite.debug) + if err != nil { + t.Fatalf("network https timeout request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/integration-test/offline-http.go b/internal/tests/integration/offline-http_test.go similarity index 88% rename from cmd/integration-test/offline-http.go rename to internal/tests/integration/offline-http_test.go index 147a1bf5a..5601722de 100644 --- a/cmd/integration-test/offline-http.go +++ b/internal/tests/integration/offline-http_test.go @@ -1,12 +1,15 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) -var offlineHttpTestcases = []TestCaseInfo{ +var offlineHttpTestcases = []integrationCase{ {Path: "protocols/offlinehttp/rfc-req-resp.yaml", TestCase: &RfcRequestResponse{}}, {Path: "protocols/offlinehttp/offline-allowed-paths.yaml", TestCase: &RequestResponseWithAllowedPaths{}}, {Path: "protocols/offlinehttp/offline-raw.yaml", TestCase: &RawRequestResponse{}}, diff --git a/cmd/integration-test/profile-loader.go b/internal/tests/integration/profile-loader_test.go similarity index 89% rename from cmd/integration-test/profile-loader.go rename to internal/tests/integration/profile-loader_test.go index 3cef723ab..4b52d30d4 100644 --- a/cmd/integration-test/profile-loader.go +++ b/internal/tests/integration/profile-loader_test.go @@ -1,13 +1,16 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/utils/errkit" ) -var profileLoaderTestcases = []TestCaseInfo{ +var profileLoaderTestcases = []integrationCase{ {Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}}, {Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}}, {Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}}, diff --git a/internal/tests/integration/runner_test.go b/internal/tests/integration/runner_test.go new file mode 100644 index 000000000..976e15a91 --- /dev/null +++ b/internal/tests/integration/runner_test.go @@ -0,0 +1,300 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "flag" + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +type integrationCase struct { + Path string + TestCase testutils.TestCase + DisableOn func() bool + Serial bool +} + +type integrationFamily struct { + Name string + Cases []integrationCase + Serial bool +} + +type integrationFailures struct { + mu sync.Mutex + items []failedIntegrationCase +} + +type failedIntegrationCase struct { + Family string + Case integrationCase + Err error +} + +type stringSliceFlag []string + +func (s *stringSliceFlag) String() string { + return strings.Join(*s, " ") +} + +func (s *stringSliceFlag) Set(value string) error { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil + } + *s = append(*s, trimmed) + return nil +} + +var ( + integrationFamilyFilter = flag.String("family", "", "run only the matching integration family") + integrationTemplateFilter = flag.String("template", "", "run only integration cases whose template path contains this value") + integrationExtraArgs stringSliceFlag +) + +const maxFailureReruns = 3 + +func init() { + flag.Var(&integrationExtraArgs, "nuclei-args", "extra argument passed to nuclei for integration tests; repeat the flag to pass multiple arguments") +} + +func TestIntegrationSuites(t *testing.T) { + families := selectedIntegrationFamilies() + if len(families) == 0 { + t.Fatalf("no integration tests matched family=%q template=%q", strings.TrimSpace(*integrationFamilyFilter), strings.TrimSpace(*integrationTemplateFilter)) + } + + failed := runIntegrationPass(t, "initial", families, debug) + if len(failed) == 0 { + return + } + + if isCI() { + for attempt := 1; attempt <= maxFailureReruns && len(failed) > 0; attempt++ { + failed = withGroupedOutput(fmt.Sprintf("Rerun failed integration tests (%d/%d)", attempt, maxFailureReruns), func() []failedIntegrationCase { + return runIntegrationPass(t, fmt.Sprintf("rerun-%d", attempt), failedCasesByFamily(failed), debug) + }) + } + } + + if len(failed) > 0 && shouldDebugFailureRerun() { + failed = withGroupedOutput("Rerun failed integration tests in debug mode", func() []failedIntegrationCase { + return runIntegrationPass(t, "debug-rerun", failedCasesByFamily(failed), true) + }) + } + + if len(failed) > 0 { + t.Fatalf("integration failures remaining after reruns:\n%s", formatIntegrationFailures(failed)) + } +} + +func integrationFamilies() []integrationFamily { + return []integrationFamily{ + {Name: "http", Cases: httpTestcases}, + {Name: "interactsh", Cases: interactshTestCases}, + {Name: "code", Cases: codeTestCases}, + {Name: "workflow", Cases: workflowTestcases}, + {Name: "headless", Cases: headlessTestcases}, + {Name: "whois", Cases: whoisTestCases}, + {Name: "library", Cases: libraryTestcases}, + {Name: "template-path", Cases: templatesPathTestCases}, + {Name: "offline-http", Cases: offlineHttpTestcases}, + {Name: "fuzz", Cases: fuzzingTestCases}, + {Name: "generic", Cases: genericTestcases}, + {Name: "dsl", Cases: dslTestcases}, + {Name: "javascript", Cases: jsTestcases}, + {Name: "matcher-status", Cases: matcherStatusTestcases}, + {Name: "exporters", Cases: exportersTestCases}, + {Name: "profile-loader", Cases: profileLoaderTestcases}, + {Name: "multi-protocol", Cases: multiProtoTestcases}, + } +} + +func runIntegrationPass(t *testing.T, label string, families []integrationFamily, debugMode bool) []failedIntegrationCase { + semaphore := make(chan struct{}, parallelism()) + failures := &integrationFailures{} + previousDebug := debug + debug = debugMode + defer func() { + debug = previousDebug + }() + + for _, family := range families { + family := family + t.Run(passFamilyName(label, family.Name), func(t *testing.T) { + runIntegrationFamily(t, semaphore, failures, family) + }) + } + return failures.Items() +} + +func runIntegrationFamily(t *testing.T, semaphore chan struct{}, failures *integrationFailures, family integrationFamily) { + t.Helper() + for _, testCase := range family.Cases { + testCase := testCase + t.Run(testNameForPath(testCase.Path), func(t *testing.T) { + if !family.Serial && !testCase.Serial { + t.Parallel() + semaphore <- struct{}{} + defer func() { <-semaphore }() + } + if err := executeIntegrationCase(testCase); err != nil { + failures.Add(failedIntegrationCase{Family: family.Name, Case: testCase, Err: err}) + t.Logf("integration case failed: %v", err) + } + }) + } +} + +func executeIntegrationCase(testCase integrationCase) error { + if testCase.DisableOn != nil && testCase.DisableOn() { + return nil + } + if needsSignedCodeTemplates(testCase.Path) { + if err := ensureSignedCodeTemplates(); err != nil { + return fmt.Errorf("failed to sign code templates: %w", err) + } + } + + var retries int + var err error + for attempt := 1; attempt <= maxFailureReruns; attempt++ { + retries++ + + err = testCase.TestCase.Execute(testCase.Path) + if err == nil { + return nil + } + } + if retries > 1 { + return fmt.Errorf("test %s failed after %d attempts: %w", testCase.Path, retries, err) + } + return err +} + +func parallelism() int { + if value := strings.TrimSpace(os.Getenv("PARALLELISM")); value != "" { + if parsed, err := strconv.Atoi(value); err == nil && parsed > 0 { + return parsed + } + } + parallelism := max(min(runtime.GOMAXPROCS(0), 4), 1) + return parallelism +} + +func needsSignedCodeTemplates(path string) bool { + return strings.HasPrefix(path, "protocols/code/") || strings.Contains(path, "workflow/code-") +} + +func testNameForPath(path string) string { + replacer := strings.NewReplacer("/", "_", "\\", "_", ",", "__", ".yaml", "", ".yml", "", ".json", "", ".", "_") + name := replacer.Replace(path) + name = strings.Trim(name, "_") + if name == "" { + return "case" + } + return fmt.Sprintf("%s", name) +} + +func (f *integrationFailures) Add(failure failedIntegrationCase) { + if failure.Err == nil { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.items = append(f.items, failure) +} + +func (f *integrationFailures) Items() []failedIntegrationCase { + f.mu.Lock() + defer f.mu.Unlock() + items := make([]failedIntegrationCase, len(f.items)) + copy(items, f.items) + return items +} + +func selectedIntegrationFamilies() []integrationFamily { + familyFilter := strings.ToLower(strings.TrimSpace(*integrationFamilyFilter)) + templateFilter := strings.ToLower(strings.TrimSpace(*integrationTemplateFilter)) + selected := make([]integrationFamily, 0) + for _, family := range integrationFamilies() { + if familyFilter != "" && !strings.EqualFold(family.Name, familyFilter) { + continue + } + if templateFilter == "" { + selected = append(selected, family) + continue + } + filteredCases := make([]integrationCase, 0, len(family.Cases)) + for _, testCase := range family.Cases { + if strings.Contains(strings.ToLower(testCase.Path), templateFilter) { + filteredCases = append(filteredCases, testCase) + } + } + if len(filteredCases) == 0 { + continue + } + family.Cases = filteredCases + selected = append(selected, family) + } + return selected +} + +func failedCasesByFamily(failed []failedIntegrationCase) []integrationFamily { + orderedFamilies := make([]string, 0) + byFamily := make(map[string][]integrationCase) + for _, failure := range failed { + if _, ok := byFamily[failure.Family]; !ok { + orderedFamilies = append(orderedFamilies, failure.Family) + } + byFamily[failure.Family] = append(byFamily[failure.Family], failure.Case) + } + families := make([]integrationFamily, 0, len(orderedFamilies)) + for _, familyName := range orderedFamilies { + serial := false + for _, family := range integrationFamilies() { + if family.Name == familyName { + serial = family.Serial + break + } + } + families = append(families, integrationFamily{Name: familyName, Cases: byFamily[familyName], Serial: serial}) + } + return families +} + +func passFamilyName(label, family string) string { + if label == "" || label == "initial" { + return family + } + return label + "/" + family +} + +func shouldDebugFailureRerun() bool { + return envTruthy("DEBUG") || envTruthy("ACTIONS_STEP_DEBUG") +} + +func withGroupedOutput[T any](title string, fn func() T) T { + if isCI() { + fmt.Printf("::group::%s\n", title) + defer fmt.Println("::endgroup::") + } + return fn() +} + +func formatIntegrationFailures(failed []failedIntegrationCase) string { + lines := make([]string, 0, len(failed)) + for _, failure := range failed { + lines = append(lines, fmt.Sprintf("- [%s] %s: %v", failure.Family, failure.Case.Path, failure.Err)) + } + return strings.Join(lines, "\n") +} diff --git a/internal/tests/integration/ssl_test.go b/internal/tests/integration/ssl_test.go new file mode 100644 index 000000000..70ed356e3 --- /dev/null +++ b/internal/tests/integration/ssl_test.go @@ -0,0 +1,95 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "crypto/tls" + "net" + "testing" + + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func TestSSL(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{}) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/basic.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("basic ssl request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("BasicZTLS", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{}) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/basic-ztls.yaml", server.URL, suite.debug, "-ztls") + if err != nil { + t.Fatalf("basic ztls request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CustomCipher", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}}) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/custom-cipher.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("custom cipher ssl request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CustomVersion", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12}) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/custom-version.yaml", server.URL, suite.debug) + if err != nil { + t.Fatalf("custom version ssl request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("WithVars", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{}) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/ssl-with-vars.yaml", server.URL, suite.debug, "-V", "test=asdasdas") + if err != nil { + t.Fatalf("ssl with vars request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("MultiReq", func(t *testing.T) { + server := newTLSServer(t, &tls.Config{ + //nolint:staticcheck // SSLv3 is intentionally used for testing purposes. + MinVersion: tls.VersionSSL30, + MaxVersion: tls.VersionTLS11, + }) + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/multi-req.yaml", server.URL, suite.debug, "-V") + if err != nil { + t.Fatalf("ssl multi request failed: %v", err) + } + if err := expectResultsCount(results, 2); err != nil { + t.Fatal(err) + } + }) +} + +func newTLSServer(t *testing.T, config *tls.Config) *testutils.TCPServer { + t.Helper() + server := testutils.NewTCPServer(config, 0, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + data := make([]byte, 4) + _, _ = conn.Read(data) + }) + t.Cleanup(server.Close) + return server +} diff --git a/cmd/integration-test/template-path.go b/internal/tests/integration/template-path_test.go similarity index 91% rename from cmd/integration-test/template-path.go rename to internal/tests/integration/template-path_test.go index 0dbaa60e7..0682e3416 100644 --- a/cmd/integration-test/template-path.go +++ b/internal/tests/integration/template-path_test.go @@ -1,18 +1,21 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" "strings" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" ) func getTemplatePath() string { return config.DefaultConfig.TemplatesDirectory } -var templatesPathTestCases = []TestCaseInfo{ +var templatesPathTestCases = []integrationCase{ //template folder path issue {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd diff --git a/integration_tests/dsl/hide-version-warning.yaml b/internal/tests/integration/testdata/dsl/hide-version-warning.yaml similarity index 100% rename from integration_tests/dsl/hide-version-warning.yaml rename to internal/tests/integration/testdata/dsl/hide-version-warning.yaml diff --git a/integration_tests/dsl/show-version-warning.yaml b/internal/tests/integration/testdata/dsl/show-version-warning.yaml similarity index 100% rename from integration_tests/dsl/show-version-warning.yaml rename to internal/tests/integration/testdata/dsl/show-version-warning.yaml diff --git a/integration_tests/flow/conditional-flow-negative.yaml b/internal/tests/integration/testdata/flow/conditional-flow-negative.yaml similarity index 100% rename from integration_tests/flow/conditional-flow-negative.yaml rename to internal/tests/integration/testdata/flow/conditional-flow-negative.yaml diff --git a/integration_tests/flow/conditional-flow.yaml b/internal/tests/integration/testdata/flow/conditional-flow.yaml similarity index 100% rename from integration_tests/flow/conditional-flow.yaml rename to internal/tests/integration/testdata/flow/conditional-flow.yaml diff --git a/integration_tests/flow/dns-ns-probe.yaml b/internal/tests/integration/testdata/flow/dns-ns-probe.yaml similarity index 100% rename from integration_tests/flow/dns-ns-probe.yaml rename to internal/tests/integration/testdata/flow/dns-ns-probe.yaml diff --git a/integration_tests/flow/flow-hide-matcher.yaml b/internal/tests/integration/testdata/flow/flow-hide-matcher.yaml similarity index 100% rename from integration_tests/flow/flow-hide-matcher.yaml rename to internal/tests/integration/testdata/flow/flow-hide-matcher.yaml diff --git a/integration_tests/flow/iterate-one-value-flow.yaml b/internal/tests/integration/testdata/flow/iterate-one-value-flow.yaml similarity index 100% rename from integration_tests/flow/iterate-one-value-flow.yaml rename to internal/tests/integration/testdata/flow/iterate-one-value-flow.yaml diff --git a/integration_tests/flow/iterate-values-flow.yaml b/internal/tests/integration/testdata/flow/iterate-values-flow.yaml similarity index 100% rename from integration_tests/flow/iterate-values-flow.yaml rename to internal/tests/integration/testdata/flow/iterate-values-flow.yaml diff --git a/integration_tests/fuzz/fuzz-body-generic-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body-generic-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body-generic-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body-generic-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-body-json-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body-json-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body-json-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body-json-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body-multipart-form-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body-multipart-form-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-body-params-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body-params-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body-params-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body-params-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-body-xml-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body-xml-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body-xml-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body-xml-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-body.yaml b/internal/tests/integration/testdata/fuzz/fuzz-body.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-body.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-body.yaml diff --git a/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-cookie-error-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-cookie-error-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-cookie-error-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-headless.yaml b/internal/tests/integration/testdata/fuzz/fuzz-headless.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-headless.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-headless.yaml diff --git a/integration_tests/fuzz/fuzz-host-header-injection.yaml b/internal/tests/integration/testdata/fuzz/fuzz-host-header-injection.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-host-header-injection.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-host-header-injection.yaml diff --git a/integration_tests/fuzz/fuzz-mode.yaml b/internal/tests/integration/testdata/fuzz/fuzz-mode.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-mode.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-mode.yaml diff --git a/integration_tests/fuzz/fuzz-multi-mode.yaml b/internal/tests/integration/testdata/fuzz/fuzz-multi-mode.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-multi-mode.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-multi-mode.yaml diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/internal/tests/integration/testdata/fuzz/fuzz-path-sqli.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-path-sqli.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-path-sqli.yaml diff --git a/integration_tests/fuzz/fuzz-query-num-replace.yaml b/internal/tests/integration/testdata/fuzz/fuzz-query-num-replace.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-query-num-replace.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-query-num-replace.yaml diff --git a/integration_tests/fuzz/fuzz-query.yaml b/internal/tests/integration/testdata/fuzz/fuzz-query.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-query.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-query.yaml diff --git a/integration_tests/fuzz/fuzz-type.yaml b/internal/tests/integration/testdata/fuzz/fuzz-type.yaml similarity index 100% rename from integration_tests/fuzz/fuzz-type.yaml rename to internal/tests/integration/testdata/fuzz/fuzz-type.yaml diff --git a/integration_tests/fuzz/testData/ginandjuice.proxify.yaml b/internal/tests/integration/testdata/fuzz/testData/ginandjuice.proxify.yaml similarity index 100% rename from integration_tests/fuzz/testData/ginandjuice.proxify.yaml rename to internal/tests/integration/testdata/fuzz/testData/ginandjuice.proxify.yaml diff --git a/integration_tests/generic/auth/certificate/assets/client.crt b/internal/tests/integration/testdata/generic/auth/certificate/assets/client.crt similarity index 100% rename from integration_tests/generic/auth/certificate/assets/client.crt rename to internal/tests/integration/testdata/generic/auth/certificate/assets/client.crt diff --git a/integration_tests/generic/auth/certificate/assets/client.key b/internal/tests/integration/testdata/generic/auth/certificate/assets/client.key similarity index 100% rename from integration_tests/generic/auth/certificate/assets/client.key rename to internal/tests/integration/testdata/generic/auth/certificate/assets/client.key diff --git a/integration_tests/generic/auth/certificate/assets/server.crt b/internal/tests/integration/testdata/generic/auth/certificate/assets/server.crt similarity index 100% rename from integration_tests/generic/auth/certificate/assets/server.crt rename to internal/tests/integration/testdata/generic/auth/certificate/assets/server.crt diff --git a/integration_tests/generic/auth/certificate/http-get.yaml b/internal/tests/integration/testdata/generic/auth/certificate/http-get.yaml similarity index 100% rename from integration_tests/generic/auth/certificate/http-get.yaml rename to internal/tests/integration/testdata/generic/auth/certificate/http-get.yaml diff --git a/integration_tests/library/test.json b/internal/tests/integration/testdata/library/test.json similarity index 100% rename from integration_tests/library/test.json rename to internal/tests/integration/testdata/library/test.json diff --git a/integration_tests/library/test.yaml b/internal/tests/integration/testdata/library/test.yaml similarity index 100% rename from integration_tests/library/test.yaml rename to internal/tests/integration/testdata/library/test.yaml diff --git a/integration_tests/loader/basic.yaml b/internal/tests/integration/testdata/loader/basic.yaml similarity index 100% rename from integration_tests/loader/basic.yaml rename to internal/tests/integration/testdata/loader/basic.yaml diff --git a/integration_tests/loader/condition-matched.yaml b/internal/tests/integration/testdata/loader/condition-matched.yaml similarity index 100% rename from integration_tests/loader/condition-matched.yaml rename to internal/tests/integration/testdata/loader/condition-matched.yaml diff --git a/integration_tests/loader/excluded-template.yaml b/internal/tests/integration/testdata/loader/excluded-template.yaml similarity index 100% rename from integration_tests/loader/excluded-template.yaml rename to internal/tests/integration/testdata/loader/excluded-template.yaml diff --git a/integration_tests/loader/get-headers.yaml b/internal/tests/integration/testdata/loader/get-headers.yaml similarity index 100% rename from integration_tests/loader/get-headers.yaml rename to internal/tests/integration/testdata/loader/get-headers.yaml diff --git a/integration_tests/loader/get.yaml b/internal/tests/integration/testdata/loader/get.yaml similarity index 100% rename from integration_tests/loader/get.yaml rename to internal/tests/integration/testdata/loader/get.yaml diff --git a/integration_tests/loader/template-list.yaml b/internal/tests/integration/testdata/loader/template-list.yaml similarity index 100% rename from integration_tests/loader/template-list.yaml rename to internal/tests/integration/testdata/loader/template-list.yaml diff --git a/integration_tests/loader/workflow-list.yaml b/internal/tests/integration/testdata/loader/workflow-list.yaml similarity index 100% rename from integration_tests/loader/workflow-list.yaml rename to internal/tests/integration/testdata/loader/workflow-list.yaml diff --git a/integration_tests/profile-loader/basic.yml b/internal/tests/integration/testdata/profile-loader/basic.yml similarity index 100% rename from integration_tests/profile-loader/basic.yml rename to internal/tests/integration/testdata/profile-loader/basic.yml diff --git a/integration_tests/protocols/code/pre-condition.yaml b/internal/tests/integration/testdata/protocols/code/pre-condition.yaml similarity index 100% rename from integration_tests/protocols/code/pre-condition.yaml rename to internal/tests/integration/testdata/protocols/code/pre-condition.yaml diff --git a/integration_tests/protocols/code/ps1-snippet.yaml b/internal/tests/integration/testdata/protocols/code/ps1-snippet.yaml similarity index 100% rename from integration_tests/protocols/code/ps1-snippet.yaml rename to internal/tests/integration/testdata/protocols/code/ps1-snippet.yaml diff --git a/integration_tests/protocols/code/pwsh-echo.yaml b/internal/tests/integration/testdata/protocols/code/pwsh-echo.yaml similarity index 100% rename from integration_tests/protocols/code/pwsh-echo.yaml rename to internal/tests/integration/testdata/protocols/code/pwsh-echo.yaml diff --git a/integration_tests/protocols/code/py-env-var.yaml b/internal/tests/integration/testdata/protocols/code/py-env-var.yaml similarity index 100% rename from integration_tests/protocols/code/py-env-var.yaml rename to internal/tests/integration/testdata/protocols/code/py-env-var.yaml diff --git a/integration_tests/protocols/code/py-file.yaml b/internal/tests/integration/testdata/protocols/code/py-file.yaml similarity index 59% rename from integration_tests/protocols/code/py-file.yaml rename to internal/tests/integration/testdata/protocols/code/py-file.yaml index 3c7e00bca..b285031af 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/internal/tests/integration/testdata/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022032b81e8bb7475abf27639b0ced71355497166d664698021f26498e7031d62a23022100e99ccde578bfc0b658f16427ae9a3d18922849d3ba3e022032ea0d2a8e77fadb:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a00473045022100bb6b75819ea62be7415adf544e208fcff3501277f6f7a152e1c3083c476126f502201db7978917bc31c755a4ce53bdacc2c6cbaea1d01117e4ecbafa09117a215260:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/internal/tests/integration/testdata/protocols/code/py-interactsh.yaml similarity index 100% rename from integration_tests/protocols/code/py-interactsh.yaml rename to internal/tests/integration/testdata/protocols/code/py-interactsh.yaml diff --git a/integration_tests/protocols/code/py-nosig.yaml b/internal/tests/integration/testdata/protocols/code/py-nosig.yaml similarity index 100% rename from integration_tests/protocols/code/py-nosig.yaml rename to internal/tests/integration/testdata/protocols/code/py-nosig.yaml diff --git a/integration_tests/protocols/code/py-snippet.yaml b/internal/tests/integration/testdata/protocols/code/py-snippet.yaml similarity index 100% rename from integration_tests/protocols/code/py-snippet.yaml rename to internal/tests/integration/testdata/protocols/code/py-snippet.yaml diff --git a/integration_tests/protocols/code/py-virtual.yaml b/internal/tests/integration/testdata/protocols/code/py-virtual.yaml similarity index 100% rename from integration_tests/protocols/code/py-virtual.yaml rename to internal/tests/integration/testdata/protocols/code/py-virtual.yaml diff --git a/integration_tests/protocols/code/pyfile.py b/internal/tests/integration/testdata/protocols/code/pyfile.py similarity index 100% rename from integration_tests/protocols/code/pyfile.py rename to internal/tests/integration/testdata/protocols/code/pyfile.py diff --git a/integration_tests/protocols/code/sh-virtual.yaml b/internal/tests/integration/testdata/protocols/code/sh-virtual.yaml similarity index 100% rename from integration_tests/protocols/code/sh-virtual.yaml rename to internal/tests/integration/testdata/protocols/code/sh-virtual.yaml diff --git a/integration_tests/protocols/code/unsigned.yaml b/internal/tests/integration/testdata/protocols/code/unsigned.yaml similarity index 100% rename from integration_tests/protocols/code/unsigned.yaml rename to internal/tests/integration/testdata/protocols/code/unsigned.yaml diff --git a/integration_tests/protocols/dns/a.yaml b/internal/tests/integration/testdata/protocols/dns/a.yaml similarity index 100% rename from integration_tests/protocols/dns/a.yaml rename to internal/tests/integration/testdata/protocols/dns/a.yaml diff --git a/integration_tests/protocols/dns/aaaa.yaml b/internal/tests/integration/testdata/protocols/dns/aaaa.yaml similarity index 100% rename from integration_tests/protocols/dns/aaaa.yaml rename to internal/tests/integration/testdata/protocols/dns/aaaa.yaml diff --git a/integration_tests/protocols/dns/caa.yaml b/internal/tests/integration/testdata/protocols/dns/caa.yaml similarity index 100% rename from integration_tests/protocols/dns/caa.yaml rename to internal/tests/integration/testdata/protocols/dns/caa.yaml diff --git a/integration_tests/protocols/dns/cname-fingerprint.yaml b/internal/tests/integration/testdata/protocols/dns/cname-fingerprint.yaml similarity index 100% rename from integration_tests/protocols/dns/cname-fingerprint.yaml rename to internal/tests/integration/testdata/protocols/dns/cname-fingerprint.yaml diff --git a/integration_tests/protocols/dns/cname.yaml b/internal/tests/integration/testdata/protocols/dns/cname.yaml similarity index 100% rename from integration_tests/protocols/dns/cname.yaml rename to internal/tests/integration/testdata/protocols/dns/cname.yaml diff --git a/integration_tests/protocols/dns/dsl-matcher-variable.yaml b/internal/tests/integration/testdata/protocols/dns/dsl-matcher-variable.yaml similarity index 100% rename from integration_tests/protocols/dns/dsl-matcher-variable.yaml rename to internal/tests/integration/testdata/protocols/dns/dsl-matcher-variable.yaml diff --git a/integration_tests/protocols/dns/ns.yaml b/internal/tests/integration/testdata/protocols/dns/ns.yaml similarity index 100% rename from integration_tests/protocols/dns/ns.yaml rename to internal/tests/integration/testdata/protocols/dns/ns.yaml diff --git a/integration_tests/protocols/dns/payload.yaml b/internal/tests/integration/testdata/protocols/dns/payload.yaml similarity index 100% rename from integration_tests/protocols/dns/payload.yaml rename to internal/tests/integration/testdata/protocols/dns/payload.yaml diff --git a/integration_tests/protocols/dns/ptr.yaml b/internal/tests/integration/testdata/protocols/dns/ptr.yaml similarity index 100% rename from integration_tests/protocols/dns/ptr.yaml rename to internal/tests/integration/testdata/protocols/dns/ptr.yaml diff --git a/integration_tests/protocols/dns/srv.yaml b/internal/tests/integration/testdata/protocols/dns/srv.yaml similarity index 100% rename from integration_tests/protocols/dns/srv.yaml rename to internal/tests/integration/testdata/protocols/dns/srv.yaml diff --git a/integration_tests/protocols/dns/tlsa.yaml b/internal/tests/integration/testdata/protocols/dns/tlsa.yaml similarity index 100% rename from integration_tests/protocols/dns/tlsa.yaml rename to internal/tests/integration/testdata/protocols/dns/tlsa.yaml diff --git a/integration_tests/protocols/dns/txt.yaml b/internal/tests/integration/testdata/protocols/dns/txt.yaml similarity index 100% rename from integration_tests/protocols/dns/txt.yaml rename to internal/tests/integration/testdata/protocols/dns/txt.yaml diff --git a/integration_tests/protocols/dns/variables.yaml b/internal/tests/integration/testdata/protocols/dns/variables.yaml similarity index 100% rename from integration_tests/protocols/dns/variables.yaml rename to internal/tests/integration/testdata/protocols/dns/variables.yaml diff --git a/integration_tests/protocols/file/data/test1.txt b/internal/tests/integration/testdata/protocols/file/data/test1.txt similarity index 100% rename from integration_tests/protocols/file/data/test1.txt rename to internal/tests/integration/testdata/protocols/file/data/test1.txt diff --git a/integration_tests/protocols/file/data/test2.txt b/internal/tests/integration/testdata/protocols/file/data/test2.txt similarity index 100% rename from integration_tests/protocols/file/data/test2.txt rename to internal/tests/integration/testdata/protocols/file/data/test2.txt diff --git a/integration_tests/protocols/file/data/test3.txt b/internal/tests/integration/testdata/protocols/file/data/test3.txt similarity index 100% rename from integration_tests/protocols/file/data/test3.txt rename to internal/tests/integration/testdata/protocols/file/data/test3.txt diff --git a/integration_tests/protocols/file/extract.yaml b/internal/tests/integration/testdata/protocols/file/extract.yaml similarity index 100% rename from integration_tests/protocols/file/extract.yaml rename to internal/tests/integration/testdata/protocols/file/extract.yaml diff --git a/integration_tests/protocols/file/matcher-with-and.yaml b/internal/tests/integration/testdata/protocols/file/matcher-with-and.yaml similarity index 100% rename from integration_tests/protocols/file/matcher-with-and.yaml rename to internal/tests/integration/testdata/protocols/file/matcher-with-and.yaml diff --git a/integration_tests/protocols/file/matcher-with-nested-and.yaml b/internal/tests/integration/testdata/protocols/file/matcher-with-nested-and.yaml similarity index 100% rename from integration_tests/protocols/file/matcher-with-nested-and.yaml rename to internal/tests/integration/testdata/protocols/file/matcher-with-nested-and.yaml diff --git a/integration_tests/protocols/file/matcher-with-or.yaml b/internal/tests/integration/testdata/protocols/file/matcher-with-or.yaml similarity index 100% rename from integration_tests/protocols/file/matcher-with-or.yaml rename to internal/tests/integration/testdata/protocols/file/matcher-with-or.yaml diff --git a/integration_tests/protocols/headless/file-upload-negative.yaml b/internal/tests/integration/testdata/protocols/headless/file-upload-negative.yaml similarity index 100% rename from integration_tests/protocols/headless/file-upload-negative.yaml rename to internal/tests/integration/testdata/protocols/headless/file-upload-negative.yaml diff --git a/integration_tests/protocols/headless/file-upload.yaml b/internal/tests/integration/testdata/protocols/headless/file-upload.yaml similarity index 100% rename from integration_tests/protocols/headless/file-upload.yaml rename to internal/tests/integration/testdata/protocols/headless/file-upload.yaml diff --git a/integration_tests/protocols/headless/headless-basic.yaml b/internal/tests/integration/testdata/protocols/headless/headless-basic.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-basic.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-basic.yaml diff --git a/integration_tests/protocols/headless/headless-dsl.yaml b/internal/tests/integration/testdata/protocols/headless/headless-dsl.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-dsl.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-dsl.yaml diff --git a/integration_tests/protocols/headless/headless-extract-values.yaml b/internal/tests/integration/testdata/protocols/headless/headless-extract-values.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-extract-values.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-extract-values.yaml diff --git a/integration_tests/protocols/headless/headless-header-action.yaml b/internal/tests/integration/testdata/protocols/headless/headless-header-action.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-header-action.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-header-action.yaml diff --git a/integration_tests/protocols/headless/headless-header-status-test.yaml b/internal/tests/integration/testdata/protocols/headless/headless-header-status-test.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-header-status-test.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-header-status-test.yaml diff --git a/integration_tests/protocols/headless/headless-local.yaml b/internal/tests/integration/testdata/protocols/headless/headless-local.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-local.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-local.yaml diff --git a/integration_tests/protocols/headless/headless-payloads.yaml b/internal/tests/integration/testdata/protocols/headless/headless-payloads.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-payloads.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-payloads.yaml diff --git a/integration_tests/protocols/headless/headless-self-contained.yaml b/internal/tests/integration/testdata/protocols/headless/headless-self-contained.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-self-contained.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-self-contained.yaml diff --git a/integration_tests/protocols/headless/headless-waitevent.yaml b/internal/tests/integration/testdata/protocols/headless/headless-waitevent.yaml similarity index 100% rename from integration_tests/protocols/headless/headless-waitevent.yaml rename to internal/tests/integration/testdata/protocols/headless/headless-waitevent.yaml diff --git a/integration_tests/protocols/headless/variables.yaml b/internal/tests/integration/testdata/protocols/headless/variables.yaml similarity index 100% rename from integration_tests/protocols/headless/variables.yaml rename to internal/tests/integration/testdata/protocols/headless/variables.yaml diff --git a/integration_tests/protocols/http/annotation-timeout.yaml b/internal/tests/integration/testdata/protocols/http/annotation-timeout.yaml similarity index 100% rename from integration_tests/protocols/http/annotation-timeout.yaml rename to internal/tests/integration/testdata/protocols/http/annotation-timeout.yaml diff --git a/integration_tests/protocols/http/cl-body-with-header.yaml b/internal/tests/integration/testdata/protocols/http/cl-body-with-header.yaml similarity index 100% rename from integration_tests/protocols/http/cl-body-with-header.yaml rename to internal/tests/integration/testdata/protocols/http/cl-body-with-header.yaml diff --git a/integration_tests/protocols/http/cl-body-without-header.yaml b/internal/tests/integration/testdata/protocols/http/cl-body-without-header.yaml similarity index 100% rename from integration_tests/protocols/http/cl-body-without-header.yaml rename to internal/tests/integration/testdata/protocols/http/cl-body-without-header.yaml diff --git a/integration_tests/protocols/http/cli-with-constants.yaml b/internal/tests/integration/testdata/protocols/http/cli-with-constants.yaml similarity index 100% rename from integration_tests/protocols/http/cli-with-constants.yaml rename to internal/tests/integration/testdata/protocols/http/cli-with-constants.yaml diff --git a/integration_tests/protocols/http/constants-with-threads.yaml b/internal/tests/integration/testdata/protocols/http/constants-with-threads.yaml similarity index 100% rename from integration_tests/protocols/http/constants-with-threads.yaml rename to internal/tests/integration/testdata/protocols/http/constants-with-threads.yaml diff --git a/integration_tests/protocols/http/custom-attack-type.yaml b/internal/tests/integration/testdata/protocols/http/custom-attack-type.yaml similarity index 100% rename from integration_tests/protocols/http/custom-attack-type.yaml rename to internal/tests/integration/testdata/protocols/http/custom-attack-type.yaml diff --git a/integration_tests/protocols/http/default-matcher-condition.yaml b/internal/tests/integration/testdata/protocols/http/default-matcher-condition.yaml similarity index 100% rename from integration_tests/protocols/http/default-matcher-condition.yaml rename to internal/tests/integration/testdata/protocols/http/default-matcher-condition.yaml diff --git a/integration_tests/protocols/http/disable-path-automerge.yaml b/internal/tests/integration/testdata/protocols/http/disable-path-automerge.yaml similarity index 100% rename from integration_tests/protocols/http/disable-path-automerge.yaml rename to internal/tests/integration/testdata/protocols/http/disable-path-automerge.yaml diff --git a/integration_tests/protocols/http/disable-redirects.yaml b/internal/tests/integration/testdata/protocols/http/disable-redirects.yaml similarity index 100% rename from integration_tests/protocols/http/disable-redirects.yaml rename to internal/tests/integration/testdata/protocols/http/disable-redirects.yaml diff --git a/integration_tests/protocols/http/dsl-functions.yaml b/internal/tests/integration/testdata/protocols/http/dsl-functions.yaml similarity index 100% rename from integration_tests/protocols/http/dsl-functions.yaml rename to internal/tests/integration/testdata/protocols/http/dsl-functions.yaml diff --git a/integration_tests/protocols/http/dsl-matcher-variable.yaml b/internal/tests/integration/testdata/protocols/http/dsl-matcher-variable.yaml similarity index 100% rename from integration_tests/protocols/http/dsl-matcher-variable.yaml rename to internal/tests/integration/testdata/protocols/http/dsl-matcher-variable.yaml diff --git a/integration_tests/protocols/http/get-all-ips.yaml b/internal/tests/integration/testdata/protocols/http/get-all-ips.yaml similarity index 100% rename from integration_tests/protocols/http/get-all-ips.yaml rename to internal/tests/integration/testdata/protocols/http/get-all-ips.yaml diff --git a/integration_tests/protocols/http/get-case-insensitive.yaml b/internal/tests/integration/testdata/protocols/http/get-case-insensitive.yaml similarity index 100% rename from integration_tests/protocols/http/get-case-insensitive.yaml rename to internal/tests/integration/testdata/protocols/http/get-case-insensitive.yaml diff --git a/integration_tests/protocols/http/get-headers.yaml b/internal/tests/integration/testdata/protocols/http/get-headers.yaml similarity index 100% rename from integration_tests/protocols/http/get-headers.yaml rename to internal/tests/integration/testdata/protocols/http/get-headers.yaml diff --git a/integration_tests/protocols/http/get-host-redirects.yaml b/internal/tests/integration/testdata/protocols/http/get-host-redirects.yaml similarity index 100% rename from integration_tests/protocols/http/get-host-redirects.yaml rename to internal/tests/integration/testdata/protocols/http/get-host-redirects.yaml diff --git a/integration_tests/protocols/http/get-override-sni.yaml b/internal/tests/integration/testdata/protocols/http/get-override-sni.yaml similarity index 100% rename from integration_tests/protocols/http/get-override-sni.yaml rename to internal/tests/integration/testdata/protocols/http/get-override-sni.yaml diff --git a/integration_tests/protocols/http/get-query-string.yaml b/internal/tests/integration/testdata/protocols/http/get-query-string.yaml similarity index 100% rename from integration_tests/protocols/http/get-query-string.yaml rename to internal/tests/integration/testdata/protocols/http/get-query-string.yaml diff --git a/integration_tests/protocols/http/get-redirects-chain-headers.yaml b/internal/tests/integration/testdata/protocols/http/get-redirects-chain-headers.yaml similarity index 100% rename from integration_tests/protocols/http/get-redirects-chain-headers.yaml rename to internal/tests/integration/testdata/protocols/http/get-redirects-chain-headers.yaml diff --git a/integration_tests/protocols/http/get-redirects.yaml b/internal/tests/integration/testdata/protocols/http/get-redirects.yaml similarity index 100% rename from integration_tests/protocols/http/get-redirects.yaml rename to internal/tests/integration/testdata/protocols/http/get-redirects.yaml diff --git a/integration_tests/protocols/http/get-sni-unsafe.yaml b/internal/tests/integration/testdata/protocols/http/get-sni-unsafe.yaml similarity index 100% rename from integration_tests/protocols/http/get-sni-unsafe.yaml rename to internal/tests/integration/testdata/protocols/http/get-sni-unsafe.yaml diff --git a/integration_tests/protocols/http/get-sni.yaml b/internal/tests/integration/testdata/protocols/http/get-sni.yaml similarity index 100% rename from integration_tests/protocols/http/get-sni.yaml rename to internal/tests/integration/testdata/protocols/http/get-sni.yaml diff --git a/integration_tests/protocols/http/get-without-scheme.yaml b/internal/tests/integration/testdata/protocols/http/get-without-scheme.yaml similarity index 100% rename from integration_tests/protocols/http/get-without-scheme.yaml rename to internal/tests/integration/testdata/protocols/http/get-without-scheme.yaml diff --git a/integration_tests/protocols/http/get.yaml b/internal/tests/integration/testdata/protocols/http/get.yaml similarity index 100% rename from integration_tests/protocols/http/get.yaml rename to internal/tests/integration/testdata/protocols/http/get.yaml diff --git a/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml b/internal/tests/integration/testdata/protocols/http/http-matcher-extractor-dy-extractor.yaml similarity index 100% rename from integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml rename to internal/tests/integration/testdata/protocols/http/http-matcher-extractor-dy-extractor.yaml diff --git a/integration_tests/protocols/http/http-paths.yaml b/internal/tests/integration/testdata/protocols/http/http-paths.yaml similarity index 100% rename from integration_tests/protocols/http/http-paths.yaml rename to internal/tests/integration/testdata/protocols/http/http-paths.yaml diff --git a/integration_tests/protocols/http/http-preprocessor.yaml b/internal/tests/integration/testdata/protocols/http/http-preprocessor.yaml similarity index 100% rename from integration_tests/protocols/http/http-preprocessor.yaml rename to internal/tests/integration/testdata/protocols/http/http-preprocessor.yaml diff --git a/integration_tests/protocols/http/interactsh-requests-mc-and.yaml b/internal/tests/integration/testdata/protocols/http/interactsh-requests-mc-and.yaml similarity index 100% rename from integration_tests/protocols/http/interactsh-requests-mc-and.yaml rename to internal/tests/integration/testdata/protocols/http/interactsh-requests-mc-and.yaml diff --git a/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml b/internal/tests/integration/testdata/protocols/http/interactsh-stop-at-first-match.yaml similarity index 100% rename from integration_tests/protocols/http/interactsh-stop-at-first-match.yaml rename to internal/tests/integration/testdata/protocols/http/interactsh-stop-at-first-match.yaml diff --git a/integration_tests/protocols/http/interactsh-with-payloads.yaml b/internal/tests/integration/testdata/protocols/http/interactsh-with-payloads.yaml similarity index 100% rename from integration_tests/protocols/http/interactsh-with-payloads.yaml rename to internal/tests/integration/testdata/protocols/http/interactsh-with-payloads.yaml diff --git a/integration_tests/protocols/http/interactsh.yaml b/internal/tests/integration/testdata/protocols/http/interactsh.yaml similarity index 100% rename from integration_tests/protocols/http/interactsh.yaml rename to internal/tests/integration/testdata/protocols/http/interactsh.yaml diff --git a/integration_tests/protocols/http/matcher-status-and-cluster.yaml b/internal/tests/integration/testdata/protocols/http/matcher-status-and-cluster.yaml similarity index 100% rename from integration_tests/protocols/http/matcher-status-and-cluster.yaml rename to internal/tests/integration/testdata/protocols/http/matcher-status-and-cluster.yaml diff --git a/integration_tests/protocols/http/matcher-status-and.yaml b/internal/tests/integration/testdata/protocols/http/matcher-status-and.yaml similarity index 100% rename from integration_tests/protocols/http/matcher-status-and.yaml rename to internal/tests/integration/testdata/protocols/http/matcher-status-and.yaml diff --git a/integration_tests/protocols/http/matcher-status.yaml b/internal/tests/integration/testdata/protocols/http/matcher-status.yaml similarity index 100% rename from integration_tests/protocols/http/matcher-status.yaml rename to internal/tests/integration/testdata/protocols/http/matcher-status.yaml diff --git a/integration_tests/protocols/http/multi-http-var-sharing.yaml b/internal/tests/integration/testdata/protocols/http/multi-http-var-sharing.yaml similarity index 100% rename from integration_tests/protocols/http/multi-http-var-sharing.yaml rename to internal/tests/integration/testdata/protocols/http/multi-http-var-sharing.yaml diff --git a/integration_tests/protocols/http/multi-request.yaml b/internal/tests/integration/testdata/protocols/http/multi-request.yaml similarity index 100% rename from integration_tests/protocols/http/multi-request.yaml rename to internal/tests/integration/testdata/protocols/http/multi-request.yaml diff --git a/integration_tests/protocols/http/post-body.yaml b/internal/tests/integration/testdata/protocols/http/post-body.yaml similarity index 100% rename from integration_tests/protocols/http/post-body.yaml rename to internal/tests/integration/testdata/protocols/http/post-body.yaml diff --git a/integration_tests/protocols/http/post-json-body.yaml b/internal/tests/integration/testdata/protocols/http/post-json-body.yaml similarity index 100% rename from integration_tests/protocols/http/post-json-body.yaml rename to internal/tests/integration/testdata/protocols/http/post-json-body.yaml diff --git a/integration_tests/protocols/http/post-multipart-body.yaml b/internal/tests/integration/testdata/protocols/http/post-multipart-body.yaml similarity index 100% rename from integration_tests/protocols/http/post-multipart-body.yaml rename to internal/tests/integration/testdata/protocols/http/post-multipart-body.yaml diff --git a/integration_tests/protocols/http/race-condition-with-delay.yaml b/internal/tests/integration/testdata/protocols/http/race-condition-with-delay.yaml similarity index 100% rename from integration_tests/protocols/http/race-condition-with-delay.yaml rename to internal/tests/integration/testdata/protocols/http/race-condition-with-delay.yaml diff --git a/integration_tests/protocols/http/race-multiple.yaml b/internal/tests/integration/testdata/protocols/http/race-multiple.yaml similarity index 100% rename from integration_tests/protocols/http/race-multiple.yaml rename to internal/tests/integration/testdata/protocols/http/race-multiple.yaml diff --git a/integration_tests/protocols/http/race-simple.yaml b/internal/tests/integration/testdata/protocols/http/race-simple.yaml similarity index 100% rename from integration_tests/protocols/http/race-simple.yaml rename to internal/tests/integration/testdata/protocols/http/race-simple.yaml diff --git a/integration_tests/protocols/http/race-with-variables.yaml b/internal/tests/integration/testdata/protocols/http/race-with-variables.yaml similarity index 100% rename from integration_tests/protocols/http/race-with-variables.yaml rename to internal/tests/integration/testdata/protocols/http/race-with-variables.yaml diff --git a/integration_tests/protocols/http/raw-cookie-reuse.yaml b/internal/tests/integration/testdata/protocols/http/raw-cookie-reuse.yaml similarity index 100% rename from integration_tests/protocols/http/raw-cookie-reuse.yaml rename to internal/tests/integration/testdata/protocols/http/raw-cookie-reuse.yaml diff --git a/integration_tests/protocols/http/raw-dynamic-extractor.yaml b/internal/tests/integration/testdata/protocols/http/raw-dynamic-extractor.yaml similarity index 100% rename from integration_tests/protocols/http/raw-dynamic-extractor.yaml rename to internal/tests/integration/testdata/protocols/http/raw-dynamic-extractor.yaml diff --git a/integration_tests/protocols/http/raw-get-query.yaml b/internal/tests/integration/testdata/protocols/http/raw-get-query.yaml similarity index 100% rename from integration_tests/protocols/http/raw-get-query.yaml rename to internal/tests/integration/testdata/protocols/http/raw-get-query.yaml diff --git a/integration_tests/protocols/http/raw-get.yaml b/internal/tests/integration/testdata/protocols/http/raw-get.yaml similarity index 100% rename from integration_tests/protocols/http/raw-get.yaml rename to internal/tests/integration/testdata/protocols/http/raw-get.yaml diff --git a/integration_tests/protocols/http/raw-path-single-slash.yaml b/internal/tests/integration/testdata/protocols/http/raw-path-single-slash.yaml similarity index 100% rename from integration_tests/protocols/http/raw-path-single-slash.yaml rename to internal/tests/integration/testdata/protocols/http/raw-path-single-slash.yaml diff --git a/integration_tests/protocols/http/raw-path-trailing-slash.yaml b/internal/tests/integration/testdata/protocols/http/raw-path-trailing-slash.yaml similarity index 100% rename from integration_tests/protocols/http/raw-path-trailing-slash.yaml rename to internal/tests/integration/testdata/protocols/http/raw-path-trailing-slash.yaml diff --git a/integration_tests/protocols/http/raw-payload.yaml b/internal/tests/integration/testdata/protocols/http/raw-payload.yaml similarity index 100% rename from integration_tests/protocols/http/raw-payload.yaml rename to internal/tests/integration/testdata/protocols/http/raw-payload.yaml diff --git a/integration_tests/protocols/http/raw-post-body.yaml b/internal/tests/integration/testdata/protocols/http/raw-post-body.yaml similarity index 100% rename from integration_tests/protocols/http/raw-post-body.yaml rename to internal/tests/integration/testdata/protocols/http/raw-post-body.yaml diff --git a/integration_tests/protocols/http/raw-unsafe-path-single-slash.yaml b/internal/tests/integration/testdata/protocols/http/raw-unsafe-path-single-slash.yaml similarity index 100% rename from integration_tests/protocols/http/raw-unsafe-path-single-slash.yaml rename to internal/tests/integration/testdata/protocols/http/raw-unsafe-path-single-slash.yaml diff --git a/integration_tests/protocols/http/raw-unsafe-path.yaml b/internal/tests/integration/testdata/protocols/http/raw-unsafe-path.yaml similarity index 100% rename from integration_tests/protocols/http/raw-unsafe-path.yaml rename to internal/tests/integration/testdata/protocols/http/raw-unsafe-path.yaml diff --git a/integration_tests/protocols/http/raw-unsafe-request.yaml b/internal/tests/integration/testdata/protocols/http/raw-unsafe-request.yaml similarity index 100% rename from integration_tests/protocols/http/raw-unsafe-request.yaml rename to internal/tests/integration/testdata/protocols/http/raw-unsafe-request.yaml diff --git a/integration_tests/protocols/http/raw-unsafe-with-params.yaml b/internal/tests/integration/testdata/protocols/http/raw-unsafe-with-params.yaml similarity index 100% rename from integration_tests/protocols/http/raw-unsafe-with-params.yaml rename to internal/tests/integration/testdata/protocols/http/raw-unsafe-with-params.yaml diff --git a/integration_tests/protocols/http/raw-with-params.yaml b/internal/tests/integration/testdata/protocols/http/raw-with-params.yaml similarity index 100% rename from integration_tests/protocols/http/raw-with-params.yaml rename to internal/tests/integration/testdata/protocols/http/raw-with-params.yaml diff --git a/integration_tests/protocols/http/redirect-match-url.yaml b/internal/tests/integration/testdata/protocols/http/redirect-match-url.yaml similarity index 100% rename from integration_tests/protocols/http/redirect-match-url.yaml rename to internal/tests/integration/testdata/protocols/http/redirect-match-url.yaml diff --git a/integration_tests/protocols/http/request-condition-new.yaml b/internal/tests/integration/testdata/protocols/http/request-condition-new.yaml similarity index 100% rename from integration_tests/protocols/http/request-condition-new.yaml rename to internal/tests/integration/testdata/protocols/http/request-condition-new.yaml diff --git a/integration_tests/protocols/http/request-condition.yaml b/internal/tests/integration/testdata/protocols/http/request-condition.yaml similarity index 100% rename from integration_tests/protocols/http/request-condition.yaml rename to internal/tests/integration/testdata/protocols/http/request-condition.yaml diff --git a/integration_tests/protocols/http/response-data-literal-reuse.yaml b/internal/tests/integration/testdata/protocols/http/response-data-literal-reuse.yaml similarity index 100% rename from integration_tests/protocols/http/response-data-literal-reuse.yaml rename to internal/tests/integration/testdata/protocols/http/response-data-literal-reuse.yaml diff --git a/integration_tests/protocols/http/self-contained-file-input.yaml b/internal/tests/integration/testdata/protocols/http/self-contained-file-input.yaml similarity index 100% rename from integration_tests/protocols/http/self-contained-file-input.yaml rename to internal/tests/integration/testdata/protocols/http/self-contained-file-input.yaml diff --git a/integration_tests/protocols/http/self-contained-with-params.yaml b/internal/tests/integration/testdata/protocols/http/self-contained-with-params.yaml similarity index 100% rename from integration_tests/protocols/http/self-contained-with-params.yaml rename to internal/tests/integration/testdata/protocols/http/self-contained-with-params.yaml diff --git a/integration_tests/protocols/http/self-contained-with-path.yaml b/internal/tests/integration/testdata/protocols/http/self-contained-with-path.yaml similarity index 100% rename from integration_tests/protocols/http/self-contained-with-path.yaml rename to internal/tests/integration/testdata/protocols/http/self-contained-with-path.yaml diff --git a/integration_tests/protocols/http/self-contained.yaml b/internal/tests/integration/testdata/protocols/http/self-contained.yaml similarity index 100% rename from integration_tests/protocols/http/self-contained.yaml rename to internal/tests/integration/testdata/protocols/http/self-contained.yaml diff --git a/integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml b/internal/tests/integration/testdata/protocols/http/stop-at-first-match-with-extractors.yaml similarity index 100% rename from integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml rename to internal/tests/integration/testdata/protocols/http/stop-at-first-match-with-extractors.yaml diff --git a/integration_tests/protocols/http/stop-at-first-match.yaml b/internal/tests/integration/testdata/protocols/http/stop-at-first-match.yaml similarity index 100% rename from integration_tests/protocols/http/stop-at-first-match.yaml rename to internal/tests/integration/testdata/protocols/http/stop-at-first-match.yaml diff --git a/integration_tests/protocols/http/variable-dsl-function.yaml b/internal/tests/integration/testdata/protocols/http/variable-dsl-function.yaml similarity index 100% rename from integration_tests/protocols/http/variable-dsl-function.yaml rename to internal/tests/integration/testdata/protocols/http/variable-dsl-function.yaml diff --git a/integration_tests/protocols/http/variables-threads-previous.yaml b/internal/tests/integration/testdata/protocols/http/variables-threads-previous.yaml similarity index 100% rename from integration_tests/protocols/http/variables-threads-previous.yaml rename to internal/tests/integration/testdata/protocols/http/variables-threads-previous.yaml diff --git a/integration_tests/protocols/http/variables.yaml b/internal/tests/integration/testdata/protocols/http/variables.yaml similarity index 100% rename from integration_tests/protocols/http/variables.yaml rename to internal/tests/integration/testdata/protocols/http/variables.yaml diff --git a/integration_tests/protocols/javascript/multi-ports.yaml b/internal/tests/integration/testdata/protocols/javascript/multi-ports.yaml similarity index 100% rename from integration_tests/protocols/javascript/multi-ports.yaml rename to internal/tests/integration/testdata/protocols/javascript/multi-ports.yaml diff --git a/integration_tests/protocols/javascript/mysql-connect.yaml b/internal/tests/integration/testdata/protocols/javascript/mysql-connect.yaml similarity index 100% rename from integration_tests/protocols/javascript/mysql-connect.yaml rename to internal/tests/integration/testdata/protocols/javascript/mysql-connect.yaml diff --git a/integration_tests/protocols/javascript/net-https.yaml b/internal/tests/integration/testdata/protocols/javascript/net-https.yaml similarity index 100% rename from integration_tests/protocols/javascript/net-https.yaml rename to internal/tests/integration/testdata/protocols/javascript/net-https.yaml diff --git a/integration_tests/protocols/javascript/net-multi-step.yaml b/internal/tests/integration/testdata/protocols/javascript/net-multi-step.yaml similarity index 100% rename from integration_tests/protocols/javascript/net-multi-step.yaml rename to internal/tests/integration/testdata/protocols/javascript/net-multi-step.yaml diff --git a/integration_tests/protocols/javascript/no-port-args.yaml b/internal/tests/integration/testdata/protocols/javascript/no-port-args.yaml similarity index 100% rename from integration_tests/protocols/javascript/no-port-args.yaml rename to internal/tests/integration/testdata/protocols/javascript/no-port-args.yaml diff --git a/integration_tests/protocols/javascript/postgres-pass-brute.yaml b/internal/tests/integration/testdata/protocols/javascript/postgres-pass-brute.yaml similarity index 100% rename from integration_tests/protocols/javascript/postgres-pass-brute.yaml rename to internal/tests/integration/testdata/protocols/javascript/postgres-pass-brute.yaml diff --git a/integration_tests/protocols/javascript/redis-pass-brute.yaml b/internal/tests/integration/testdata/protocols/javascript/redis-pass-brute.yaml similarity index 100% rename from integration_tests/protocols/javascript/redis-pass-brute.yaml rename to internal/tests/integration/testdata/protocols/javascript/redis-pass-brute.yaml diff --git a/integration_tests/protocols/javascript/rsync-test.yaml b/internal/tests/integration/testdata/protocols/javascript/rsync-test.yaml similarity index 100% rename from integration_tests/protocols/javascript/rsync-test.yaml rename to internal/tests/integration/testdata/protocols/javascript/rsync-test.yaml diff --git a/integration_tests/protocols/javascript/ssh-server-fingerprint.yaml b/internal/tests/integration/testdata/protocols/javascript/ssh-server-fingerprint.yaml similarity index 100% rename from integration_tests/protocols/javascript/ssh-server-fingerprint.yaml rename to internal/tests/integration/testdata/protocols/javascript/ssh-server-fingerprint.yaml diff --git a/integration_tests/protocols/javascript/telnet-auth-test.yaml b/internal/tests/integration/testdata/protocols/javascript/telnet-auth-test.yaml similarity index 100% rename from integration_tests/protocols/javascript/telnet-auth-test.yaml rename to internal/tests/integration/testdata/protocols/javascript/telnet-auth-test.yaml diff --git a/integration_tests/protocols/javascript/vnc-pass-brute.yaml b/internal/tests/integration/testdata/protocols/javascript/vnc-pass-brute.yaml similarity index 100% rename from integration_tests/protocols/javascript/vnc-pass-brute.yaml rename to internal/tests/integration/testdata/protocols/javascript/vnc-pass-brute.yaml diff --git a/integration_tests/protocols/keys/README.md b/internal/tests/integration/testdata/protocols/keys/README.md similarity index 100% rename from integration_tests/protocols/keys/README.md rename to internal/tests/integration/testdata/protocols/keys/README.md diff --git a/integration_tests/protocols/keys/ci-private-key.pem b/internal/tests/integration/testdata/protocols/keys/ci-private-key.pem similarity index 100% rename from integration_tests/protocols/keys/ci-private-key.pem rename to internal/tests/integration/testdata/protocols/keys/ci-private-key.pem diff --git a/integration_tests/protocols/keys/ci.crt b/internal/tests/integration/testdata/protocols/keys/ci.crt similarity index 100% rename from integration_tests/protocols/keys/ci.crt rename to internal/tests/integration/testdata/protocols/keys/ci.crt diff --git a/integration_tests/protocols/multi/dynamic-values.yaml b/internal/tests/integration/testdata/protocols/multi/dynamic-values.yaml similarity index 100% rename from integration_tests/protocols/multi/dynamic-values.yaml rename to internal/tests/integration/testdata/protocols/multi/dynamic-values.yaml diff --git a/integration_tests/protocols/multi/evaluate-variables.yaml b/internal/tests/integration/testdata/protocols/multi/evaluate-variables.yaml similarity index 100% rename from integration_tests/protocols/multi/evaluate-variables.yaml rename to internal/tests/integration/testdata/protocols/multi/evaluate-variables.yaml diff --git a/integration_tests/protocols/multi/exported-response-vars.yaml b/internal/tests/integration/testdata/protocols/multi/exported-response-vars.yaml similarity index 100% rename from integration_tests/protocols/multi/exported-response-vars.yaml rename to internal/tests/integration/testdata/protocols/multi/exported-response-vars.yaml diff --git a/integration_tests/protocols/network/basic.yaml b/internal/tests/integration/testdata/protocols/network/basic.yaml similarity index 100% rename from integration_tests/protocols/network/basic.yaml rename to internal/tests/integration/testdata/protocols/network/basic.yaml diff --git a/integration_tests/protocols/network/hex.yaml b/internal/tests/integration/testdata/protocols/network/hex.yaml similarity index 100% rename from integration_tests/protocols/network/hex.yaml rename to internal/tests/integration/testdata/protocols/network/hex.yaml diff --git a/integration_tests/protocols/network/multi-step.yaml b/internal/tests/integration/testdata/protocols/network/multi-step.yaml similarity index 100% rename from integration_tests/protocols/network/multi-step.yaml rename to internal/tests/integration/testdata/protocols/network/multi-step.yaml diff --git a/integration_tests/protocols/network/net-https-timeout.yaml b/internal/tests/integration/testdata/protocols/network/net-https-timeout.yaml similarity index 100% rename from integration_tests/protocols/network/net-https-timeout.yaml rename to internal/tests/integration/testdata/protocols/network/net-https-timeout.yaml diff --git a/integration_tests/protocols/network/net-https.yaml b/internal/tests/integration/testdata/protocols/network/net-https.yaml similarity index 100% rename from integration_tests/protocols/network/net-https.yaml rename to internal/tests/integration/testdata/protocols/network/net-https.yaml diff --git a/integration_tests/protocols/network/network-port.yaml b/internal/tests/integration/testdata/protocols/network/network-port.yaml similarity index 100% rename from integration_tests/protocols/network/network-port.yaml rename to internal/tests/integration/testdata/protocols/network/network-port.yaml diff --git a/integration_tests/protocols/network/same-address.yaml b/internal/tests/integration/testdata/protocols/network/same-address.yaml similarity index 100% rename from integration_tests/protocols/network/same-address.yaml rename to internal/tests/integration/testdata/protocols/network/same-address.yaml diff --git a/integration_tests/protocols/network/self-contained.yaml b/internal/tests/integration/testdata/protocols/network/self-contained.yaml similarity index 100% rename from integration_tests/protocols/network/self-contained.yaml rename to internal/tests/integration/testdata/protocols/network/self-contained.yaml diff --git a/integration_tests/protocols/network/variables.yaml b/internal/tests/integration/testdata/protocols/network/variables.yaml similarity index 100% rename from integration_tests/protocols/network/variables.yaml rename to internal/tests/integration/testdata/protocols/network/variables.yaml diff --git a/integration_tests/protocols/offlinehttp/data/req-resp-with-http-keywords.txt b/internal/tests/integration/testdata/protocols/offlinehttp/data/req-resp-with-http-keywords.txt similarity index 100% rename from integration_tests/protocols/offlinehttp/data/req-resp-with-http-keywords.txt rename to internal/tests/integration/testdata/protocols/offlinehttp/data/req-resp-with-http-keywords.txt diff --git a/integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml b/internal/tests/integration/testdata/protocols/offlinehttp/offline-allowed-paths.yaml similarity index 100% rename from integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml rename to internal/tests/integration/testdata/protocols/offlinehttp/offline-allowed-paths.yaml diff --git a/integration_tests/protocols/offlinehttp/offline-raw.yaml b/internal/tests/integration/testdata/protocols/offlinehttp/offline-raw.yaml similarity index 100% rename from integration_tests/protocols/offlinehttp/offline-raw.yaml rename to internal/tests/integration/testdata/protocols/offlinehttp/offline-raw.yaml diff --git a/integration_tests/protocols/offlinehttp/rfc-req-resp.yaml b/internal/tests/integration/testdata/protocols/offlinehttp/rfc-req-resp.yaml similarity index 100% rename from integration_tests/protocols/offlinehttp/rfc-req-resp.yaml rename to internal/tests/integration/testdata/protocols/offlinehttp/rfc-req-resp.yaml diff --git a/integration_tests/protocols/ssl/basic-ztls.yaml b/internal/tests/integration/testdata/protocols/ssl/basic-ztls.yaml similarity index 100% rename from integration_tests/protocols/ssl/basic-ztls.yaml rename to internal/tests/integration/testdata/protocols/ssl/basic-ztls.yaml diff --git a/integration_tests/protocols/ssl/basic.yaml b/internal/tests/integration/testdata/protocols/ssl/basic.yaml similarity index 100% rename from integration_tests/protocols/ssl/basic.yaml rename to internal/tests/integration/testdata/protocols/ssl/basic.yaml diff --git a/integration_tests/protocols/ssl/custom-cipher.yaml b/internal/tests/integration/testdata/protocols/ssl/custom-cipher.yaml similarity index 100% rename from integration_tests/protocols/ssl/custom-cipher.yaml rename to internal/tests/integration/testdata/protocols/ssl/custom-cipher.yaml diff --git a/integration_tests/protocols/ssl/custom-version.yaml b/internal/tests/integration/testdata/protocols/ssl/custom-version.yaml similarity index 100% rename from integration_tests/protocols/ssl/custom-version.yaml rename to internal/tests/integration/testdata/protocols/ssl/custom-version.yaml diff --git a/integration_tests/protocols/ssl/multi-req.yaml b/internal/tests/integration/testdata/protocols/ssl/multi-req.yaml similarity index 100% rename from integration_tests/protocols/ssl/multi-req.yaml rename to internal/tests/integration/testdata/protocols/ssl/multi-req.yaml diff --git a/integration_tests/protocols/ssl/ssl-with-vars.yaml b/internal/tests/integration/testdata/protocols/ssl/ssl-with-vars.yaml similarity index 100% rename from integration_tests/protocols/ssl/ssl-with-vars.yaml rename to internal/tests/integration/testdata/protocols/ssl/ssl-with-vars.yaml diff --git a/integration_tests/protocols/websocket/basic.yaml b/internal/tests/integration/testdata/protocols/websocket/basic.yaml similarity index 100% rename from integration_tests/protocols/websocket/basic.yaml rename to internal/tests/integration/testdata/protocols/websocket/basic.yaml diff --git a/integration_tests/protocols/websocket/cswsh.yaml b/internal/tests/integration/testdata/protocols/websocket/cswsh.yaml similarity index 100% rename from integration_tests/protocols/websocket/cswsh.yaml rename to internal/tests/integration/testdata/protocols/websocket/cswsh.yaml diff --git a/integration_tests/protocols/websocket/no-cswsh.yaml b/internal/tests/integration/testdata/protocols/websocket/no-cswsh.yaml similarity index 100% rename from integration_tests/protocols/websocket/no-cswsh.yaml rename to internal/tests/integration/testdata/protocols/websocket/no-cswsh.yaml diff --git a/integration_tests/protocols/websocket/path.yaml b/internal/tests/integration/testdata/protocols/websocket/path.yaml similarity index 100% rename from integration_tests/protocols/websocket/path.yaml rename to internal/tests/integration/testdata/protocols/websocket/path.yaml diff --git a/integration_tests/protocols/whois/basic.yaml b/internal/tests/integration/testdata/protocols/whois/basic.yaml similarity index 100% rename from integration_tests/protocols/whois/basic.yaml rename to internal/tests/integration/testdata/protocols/whois/basic.yaml diff --git a/integration_tests/subdomains.txt b/internal/tests/integration/testdata/subdomains.txt similarity index 100% rename from integration_tests/subdomains.txt rename to internal/tests/integration/testdata/subdomains.txt diff --git a/integration_tests/test-issue-tracker-config1.yaml b/internal/tests/integration/testdata/test-issue-tracker-config1.yaml similarity index 100% rename from integration_tests/test-issue-tracker-config1.yaml rename to internal/tests/integration/testdata/test-issue-tracker-config1.yaml diff --git a/internal/tests/integration/testdata/test-issue-tracker-config2.yaml b/internal/tests/integration/testdata/test-issue-tracker-config2.yaml new file mode 100644 index 000000000..04ef818ba --- /dev/null +++ b/internal/tests/integration/testdata/test-issue-tracker-config2.yaml @@ -0,0 +1,72 @@ +allow-list: + severity: + - high + - critical +deny-list: + severity: low + +# GitHub contains configuration options for GitHub issue tracker +gitHub: + # base-url is the optional self-hosted GitHub application url + base-url: https://localhost:8443/GitHub + # username is the username of the GitHub user + username: test-username + # owner is the owner name of the repository for issues. + owner: test-owner + # token is the token for GitHub account. + token: test-token + # project-name is the name of the repository. + project-name: test-project + # issue-label is the label of the created issue type + issue-label: bug + +# GitLab contains configuration options for GitLab issue tracker +gitLab: + # base-url is the optional self-hosted GitLab application url + base-url: https://localhost:8443/GitLab + # username is the username of the GitLab user + username: test-username + # token is the token for GitLab account. + token: test-token + # project-name is the name/id of the project(repository). + project-name: "1234" + # issue-label is the label of the created issue type + issue-label: bug + # duplicate-issue-check flag to enable duplicate tracking issue check. + duplicate-issue-check: true + +# Jira contains configuration options for Jira issue tracker +jira: + # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used + cloud: true + # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created + update-existing: false + # URL is the Jira application url + url: https://localhost/Jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira + account-id: test-account-id + # email is the email of the user for Jira instance + email: test@test.com + # token is the token for Jira instance or password in case of on-prem Jira + token: test-token + # project-name is the name of the project. + project-name: test-project-name + # issue-type is the name of the created issue type + issue-type: bug + +# elasticsearch contains configuration options for elasticsearch exporter +elasticsearch: + # IP for elasticsearch instance + ip: 127.0.0.1 + # Port is the port of elasticsearch instance + port: 9200 + # IndexName is the name of the elasticsearch index + index-name: nuclei + # SSL enables ssl for elasticsearch connection + ssl: false + # SSLVerification disables SSL verification for elasticsearch + ssl-verification: false + # Username for the elasticsearch instance + username: test + # Password is the password for elasticsearch instance + password: test \ No newline at end of file diff --git a/integration_tests/workflow/basic.yaml b/internal/tests/integration/testdata/workflow/basic.yaml similarity index 100% rename from integration_tests/workflow/basic.yaml rename to internal/tests/integration/testdata/workflow/basic.yaml diff --git a/integration_tests/workflow/code-template-1.yaml b/internal/tests/integration/testdata/workflow/code-template-1.yaml similarity index 100% rename from integration_tests/workflow/code-template-1.yaml rename to internal/tests/integration/testdata/workflow/code-template-1.yaml diff --git a/integration_tests/workflow/code-template-2.yaml b/internal/tests/integration/testdata/workflow/code-template-2.yaml similarity index 100% rename from integration_tests/workflow/code-template-2.yaml rename to internal/tests/integration/testdata/workflow/code-template-2.yaml diff --git a/integration_tests/workflow/code-value-share-workflow.yaml b/internal/tests/integration/testdata/workflow/code-value-share-workflow.yaml similarity index 100% rename from integration_tests/workflow/code-value-share-workflow.yaml rename to internal/tests/integration/testdata/workflow/code-value-share-workflow.yaml diff --git a/integration_tests/workflow/complex-conditions.yaml b/internal/tests/integration/testdata/workflow/complex-conditions.yaml similarity index 100% rename from integration_tests/workflow/complex-conditions.yaml rename to internal/tests/integration/testdata/workflow/complex-conditions.yaml diff --git a/integration_tests/workflow/condition-matched.yaml b/internal/tests/integration/testdata/workflow/condition-matched.yaml similarity index 100% rename from integration_tests/workflow/condition-matched.yaml rename to internal/tests/integration/testdata/workflow/condition-matched.yaml diff --git a/integration_tests/workflow/condition-unmatched.yaml b/internal/tests/integration/testdata/workflow/condition-unmatched.yaml similarity index 100% rename from integration_tests/workflow/condition-unmatched.yaml rename to internal/tests/integration/testdata/workflow/condition-unmatched.yaml diff --git a/integration_tests/workflow/dns-value-share-template-1.yaml b/internal/tests/integration/testdata/workflow/dns-value-share-template-1.yaml similarity index 100% rename from integration_tests/workflow/dns-value-share-template-1.yaml rename to internal/tests/integration/testdata/workflow/dns-value-share-template-1.yaml diff --git a/integration_tests/workflow/dns-value-share-template-2.yaml b/internal/tests/integration/testdata/workflow/dns-value-share-template-2.yaml similarity index 100% rename from integration_tests/workflow/dns-value-share-template-2.yaml rename to internal/tests/integration/testdata/workflow/dns-value-share-template-2.yaml diff --git a/integration_tests/workflow/dns-value-share-template-3.yaml b/internal/tests/integration/testdata/workflow/dns-value-share-template-3.yaml similarity index 100% rename from integration_tests/workflow/dns-value-share-template-3.yaml rename to internal/tests/integration/testdata/workflow/dns-value-share-template-3.yaml diff --git a/integration_tests/workflow/dns-value-share-workflow.yaml b/internal/tests/integration/testdata/workflow/dns-value-share-workflow.yaml similarity index 100% rename from integration_tests/workflow/dns-value-share-workflow.yaml rename to internal/tests/integration/testdata/workflow/dns-value-share-workflow.yaml diff --git a/integration_tests/workflow/headless-1.yaml b/internal/tests/integration/testdata/workflow/headless-1.yaml similarity index 100% rename from integration_tests/workflow/headless-1.yaml rename to internal/tests/integration/testdata/workflow/headless-1.yaml diff --git a/integration_tests/workflow/http-1.yaml b/internal/tests/integration/testdata/workflow/http-1.yaml similarity index 100% rename from integration_tests/workflow/http-1.yaml rename to internal/tests/integration/testdata/workflow/http-1.yaml diff --git a/integration_tests/workflow/http-2.yaml b/internal/tests/integration/testdata/workflow/http-2.yaml similarity index 100% rename from integration_tests/workflow/http-2.yaml rename to internal/tests/integration/testdata/workflow/http-2.yaml diff --git a/integration_tests/workflow/http-3.yaml b/internal/tests/integration/testdata/workflow/http-3.yaml similarity index 100% rename from integration_tests/workflow/http-3.yaml rename to internal/tests/integration/testdata/workflow/http-3.yaml diff --git a/integration_tests/workflow/http-value-share-template-1.yaml b/internal/tests/integration/testdata/workflow/http-value-share-template-1.yaml similarity index 100% rename from integration_tests/workflow/http-value-share-template-1.yaml rename to internal/tests/integration/testdata/workflow/http-value-share-template-1.yaml diff --git a/integration_tests/workflow/http-value-share-template-2.yaml b/internal/tests/integration/testdata/workflow/http-value-share-template-2.yaml similarity index 100% rename from integration_tests/workflow/http-value-share-template-2.yaml rename to internal/tests/integration/testdata/workflow/http-value-share-template-2.yaml diff --git a/integration_tests/workflow/http-value-share-workflow.yaml b/internal/tests/integration/testdata/workflow/http-value-share-workflow.yaml similarity index 100% rename from integration_tests/workflow/http-value-share-workflow.yaml rename to internal/tests/integration/testdata/workflow/http-value-share-workflow.yaml diff --git a/integration_tests/workflow/match-1.yaml b/internal/tests/integration/testdata/workflow/match-1.yaml similarity index 100% rename from integration_tests/workflow/match-1.yaml rename to internal/tests/integration/testdata/workflow/match-1.yaml diff --git a/integration_tests/workflow/match-2.yaml b/internal/tests/integration/testdata/workflow/match-2.yaml similarity index 100% rename from integration_tests/workflow/match-2.yaml rename to internal/tests/integration/testdata/workflow/match-2.yaml diff --git a/integration_tests/workflow/match-3.yaml b/internal/tests/integration/testdata/workflow/match-3.yaml similarity index 100% rename from integration_tests/workflow/match-3.yaml rename to internal/tests/integration/testdata/workflow/match-3.yaml diff --git a/integration_tests/workflow/matcher-name.yaml b/internal/tests/integration/testdata/workflow/matcher-name.yaml similarity index 100% rename from integration_tests/workflow/matcher-name.yaml rename to internal/tests/integration/testdata/workflow/matcher-name.yaml diff --git a/integration_tests/workflow/multimatch-value-share-template.yaml b/internal/tests/integration/testdata/workflow/multimatch-value-share-template.yaml similarity index 100% rename from integration_tests/workflow/multimatch-value-share-template.yaml rename to internal/tests/integration/testdata/workflow/multimatch-value-share-template.yaml diff --git a/integration_tests/workflow/multimatch-value-share-workflow.yaml b/internal/tests/integration/testdata/workflow/multimatch-value-share-workflow.yaml similarity index 100% rename from integration_tests/workflow/multimatch-value-share-workflow.yaml rename to internal/tests/integration/testdata/workflow/multimatch-value-share-workflow.yaml diff --git a/integration_tests/workflow/multiprotocol-value-share-template.yaml b/internal/tests/integration/testdata/workflow/multiprotocol-value-share-template.yaml similarity index 100% rename from integration_tests/workflow/multiprotocol-value-share-template.yaml rename to internal/tests/integration/testdata/workflow/multiprotocol-value-share-template.yaml diff --git a/integration_tests/workflow/multiprotocol-value-share-workflow.yaml b/internal/tests/integration/testdata/workflow/multiprotocol-value-share-workflow.yaml similarity index 100% rename from integration_tests/workflow/multiprotocol-value-share-workflow.yaml rename to internal/tests/integration/testdata/workflow/multiprotocol-value-share-workflow.yaml diff --git a/integration_tests/workflow/nomatch-1.yaml b/internal/tests/integration/testdata/workflow/nomatch-1.yaml similarity index 100% rename from integration_tests/workflow/nomatch-1.yaml rename to internal/tests/integration/testdata/workflow/nomatch-1.yaml diff --git a/integration_tests/workflow/shared-cookie.yaml b/internal/tests/integration/testdata/workflow/shared-cookie.yaml similarity index 100% rename from integration_tests/workflow/shared-cookie.yaml rename to internal/tests/integration/testdata/workflow/shared-cookie.yaml diff --git a/internal/tests/integration/websocket_test.go b/internal/tests/integration/websocket_test.go new file mode 100644 index 000000000..cf71c9fe3 --- /dev/null +++ b/internal/tests/integration/websocket_test.go @@ -0,0 +1,79 @@ +//go:build integration +// +build integration + +package integration_test + +import ( + "net" + "strings" + "testing" + + "github.com/gobwas/ws/wsutil" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" +) + +func TestWebSocket(t *testing.T) { + t.Run("Basic", func(t *testing.T) { + connHandler := func(conn net.Conn) { + for { + msg, op, err := wsutil.ReadClientData(conn) + if err != nil { + return + } + if string(msg) != "hello" { + return + } + _ = wsutil.WriteServerMessage(conn, op, []byte("world")) + } + } + server := testutils.NewWebsocketServer("", connHandler, func(origin string) bool { return true }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/websocket/basic.yaml", strings.ReplaceAll(server.URL, "http", "ws"), suite.debug) + if err != nil { + t.Fatalf("basic websocket request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("CSWSH", func(t *testing.T) { + server := testutils.NewWebsocketServer("", func(conn net.Conn) {}, func(origin string) bool { return true }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/websocket/cswsh.yaml", strings.ReplaceAll(server.URL, "http", "ws"), suite.debug) + if err != nil { + t.Fatalf("cswsh websocket request failed: %v", err) + } + if err := expectResultsCount(results, 1); err != nil { + t.Fatal(err) + } + }) + + t.Run("NoCSWSH", func(t *testing.T) { + server := testutils.NewWebsocketServer("", func(conn net.Conn) {}, func(origin string) bool { return origin == "https://google.com" }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/websocket/no-cswsh.yaml", strings.ReplaceAll(server.URL, "http", "ws"), suite.debug) + if err != nil { + t.Fatalf("no-cswsh websocket request failed: %v", err) + } + if err := expectResultsCount(results, 0); err != nil { + t.Fatal(err) + } + }) + + t.Run("Path", func(t *testing.T) { + server := testutils.NewWebsocketServer("/test", func(conn net.Conn) {}, func(origin string) bool { return origin == "https://google.com" }) + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults("protocols/websocket/path.yaml", strings.ReplaceAll(server.URL, "http", "ws"), suite.debug) + if err != nil { + t.Fatalf("path websocket request failed: %v", err) + } + if err := expectResultsCount(results, 0); err != nil { + t.Fatal(err) + } + }) +} diff --git a/cmd/integration-test/whois.go b/internal/tests/integration/whois_test.go similarity index 69% rename from cmd/integration-test/whois.go rename to internal/tests/integration/whois_test.go index 1d39214e9..e9aad2a41 100644 --- a/cmd/integration-test/whois.go +++ b/internal/tests/integration/whois_test.go @@ -1,10 +1,13 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) -var whoisTestCases = []TestCaseInfo{ +var whoisTestCases = []integrationCase{ {Path: "protocols/whois/basic.yaml", TestCase: &whoisBasic{}}, } diff --git a/cmd/integration-test/workflow.go b/internal/tests/integration/workflow_test.go similarity index 91% rename from cmd/integration-test/workflow.go rename to internal/tests/integration/workflow_test.go index 3032e8b59..0a6f423e2 100644 --- a/cmd/integration-test/workflow.go +++ b/internal/tests/integration/workflow_test.go @@ -1,22 +1,23 @@ -package main +//go:build integration +// +build integration + +package integration_test import ( "fmt" "io" - "log" "net/http" "net/http/httptest" "strings" "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" sliceutil "github.com/projectdiscovery/utils/slice" ) -var workflowTestcases = []TestCaseInfo{ +var workflowTestcases = []integrationCase{ {Path: "workflow/basic.yaml", TestCase: &workflowBasic{}}, {Path: "workflow/condition-matched.yaml", TestCase: &workflowConditionMatched{}}, {Path: "workflow/condition-unmatched.yaml", TestCase: &workflowConditionUnmatch{}}, @@ -30,32 +31,6 @@ var workflowTestcases = []TestCaseInfo{ {Path: "workflow/shared-cookie.yaml", TestCase: &workflowSharedCookies{}}, } -func init() { - // sign code templates (unless they are disabled) - if !isCodeDisabled() { - // allow local file access to load content of file references in template - // in order to sign them for testing purposes - templates.TemplateSignerLFA() - - // testCertFile and testKeyFile are declared in code.go - tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile) - if err != nil { - panic(err) - } - - // only the code templates are necessary to be signed - var templatesToSign = []string{ - "workflow/code-template-1.yaml", - "workflow/code-template-2.yaml", - } - for _, templatePath := range templatesToSign { - if err := templates.SignTemplate(tsigner, templatePath); err != nil { - log.Fatalf("Could not sign template %v got: %s\n", templatePath, err) - } - } - } -} - type workflowBasic struct{} // Execute executes a test case and returns an error if occurred diff --git a/pkg/testutils/testheadless/headless_local.go b/internal/tests/testheadless/headless_local.go similarity index 100% rename from pkg/testutils/testheadless/headless_local.go rename to internal/tests/testheadless/headless_local.go diff --git a/pkg/testutils/testheadless/headless_runtime.go b/internal/tests/testheadless/headless_runtime.go similarity index 91% rename from pkg/testutils/testheadless/headless_runtime.go rename to internal/tests/testheadless/headless_runtime.go index 60d141444..f42e43a85 100644 --- a/pkg/testutils/testheadless/headless_runtime.go +++ b/internal/tests/testheadless/headless_runtime.go @@ -2,5 +2,5 @@ package testheadless -// HeadlessLocal determines if local headless chrome should be used in tests +// HeadlessLocal determines if local headless chrome should be used in tests. const HeadlessLocal = false diff --git a/internal/tests/testutils/integration.go b/internal/tests/testutils/integration.go new file mode 100644 index 000000000..cad8c6a22 --- /dev/null +++ b/internal/tests/testutils/integration.go @@ -0,0 +1,484 @@ +package testutils + +import ( + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "regexp" + "strings" + + "github.com/gobwas/ws" + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/utils/conversion" +) + +type Runner struct { + BinaryPath string + WorkingDir string + BaseEnv []string + ExtraArgs []string + DisableAutoUpdate bool + AllowLocalFileAccess bool + InteractionsPollDuration string + InteractionsCooldownPeriod string +} + +type RunnerOption func(*Runner) + +func NewRunner(options ...RunnerOption) *Runner { + runner := &Runner{ + BinaryPath: "nuclei", + BaseEnv: []string{"DISABLE_CLOUD_UPLOAD_WRN=true", "DISABLE_CLOUD_UPLOAD=true"}, + DisableAutoUpdate: true, + AllowLocalFileAccess: true, + InteractionsPollDuration: "1", + InteractionsCooldownPeriod: "10", + } + for _, option := range options { + option(runner) + } + + return runner +} + +func (r *Runner) Clone(options ...RunnerOption) *Runner { + clone := &Runner{ + BinaryPath: r.BinaryPath, + WorkingDir: r.WorkingDir, + BaseEnv: append([]string{}, r.BaseEnv...), + ExtraArgs: append([]string{}, r.ExtraArgs...), + DisableAutoUpdate: r.DisableAutoUpdate, + AllowLocalFileAccess: r.AllowLocalFileAccess, + InteractionsPollDuration: r.InteractionsPollDuration, + InteractionsCooldownPeriod: r.InteractionsCooldownPeriod, + } + for _, option := range options { + option(clone) + } + + return clone +} + +func WithBinaryPath(path string) RunnerOption { + return func(r *Runner) { + if trimmed := strings.TrimSpace(path); trimmed != "" { + r.BinaryPath = trimmed + } + } +} + +func WithWorkingDir(dir string) RunnerOption { + return func(r *Runner) { + r.WorkingDir = strings.TrimSpace(dir) + } +} + +func WithBaseEnv(env ...string) RunnerOption { + return func(r *Runner) { + r.BaseEnv = append(r.BaseEnv, env...) + } +} + +func WithExtraArgs(args ...string) RunnerOption { + return func(r *Runner) { + r.ExtraArgs = append([]string{}, args...) + } +} + +var defaultRunner = NewRunner() + +func DefaultRunner() *Runner { + return defaultRunner.Clone() +} + +func SetDefaultRunner(runner *Runner) { + if runner == nil { + defaultRunner = NewRunner() + + return + } + + defaultRunner = runner.Clone() +} + +func (r *Runner) command(binaryPath string, args ...string) *exec.Cmd { + resolvedBinary := strings.TrimSpace(binaryPath) + if resolvedBinary == "" { + resolvedBinary = strings.TrimSpace(r.BinaryPath) + } + + if resolvedBinary == "" { + resolvedBinary = "nuclei" + } + + cmd := exec.Command(resolvedBinary, args...) + if r.WorkingDir != "" { + cmd.Dir = r.WorkingDir + } + + return cmd +} + +func (r *Runner) buildArgs(args []string, pollDuration string) []string { + builtArgs := append([]string{}, args...) + + builtArgs = append(builtArgs, r.ExtraArgs...) + + if r.DisableAutoUpdate { + builtArgs = append(builtArgs, "-duc") + } + + if pollDuration != "" { + builtArgs = append(builtArgs, "-interactions-poll-duration", pollDuration) + } + + if r.InteractionsCooldownPeriod != "" { + builtArgs = append(builtArgs, "-interactions-cooldown-period", r.InteractionsCooldownPeriod) + } + + if r.AllowLocalFileAccess { + builtArgs = append(builtArgs, "-allow-local-file-access") + } + + return builtArgs +} + +func (r *Runner) buildEnv(extra []string) []string { + merged := append([]string{}, os.Environ()...) + merged = append(merged, r.BaseEnv...) + merged = append(merged, extra...) + + return merged +} + +func (r *Runner) parseResults(output []byte, err error) ([]string, error) { + data := strings.TrimSpace(conversion.String(output)) + parts := make([]string, 0) + + for item := range strings.SplitSeq(data, "\n") { + if item != "" { + parts = append(parts, item) + } + } + + if (data == "" || len(parts) == 0) && err != nil { + return nil, fmt.Errorf("%w: %v", err, data) + } + + return parts, nil +} + +func (r *Runner) TemplateResults(template, url string, debug bool, extra ...string) ([]string, error) { + args := []string{"-t", template, "-target", url} + args = append(args, extra...) + + return r.BareResults(debug, nil, args...) +} + +func (r *Runner) WorkflowResults(workflow, url string, debug bool, extra ...string) ([]string, error) { + args := []string{"-w", workflow, "-target", url} + args = append(args, extra...) + + return r.BareResults(debug, nil, args...) +} + +func (r *Runner) BareResults(debug bool, env []string, args ...string) ([]string, error) { + cmd := r.command("") + cmd.Args = append(cmd.Args[:1], r.buildArgs(args, r.InteractionsPollDuration)...) + + cmd.Env = r.buildEnv(env) + + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + + output, err := cmd.Output() + if debug && len(output) > 0 { + fmt.Println(strings.TrimSpace(conversion.String(output))) + } + + return r.parseResults(output, err) +} + +func (r *Runner) ArgsResults(debug bool, args ...string) ([]string, error) { + cmd := r.command("", append(append([]string{}, args...), r.ExtraArgs...)...) + + cmd.Env = r.buildEnv(nil) + + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + + output, err := cmd.Output() + if debug && len(output) > 0 { + fmt.Println(strings.TrimSpace(conversion.String(output))) + } + + return r.parseResults(output, err) +} + +func (r *Runner) ArgsErrors(debug bool, env []string, args ...string) ([]string, error) { + cmd := r.command("") + cmd.Args = append(cmd.Args[:1], r.buildArgs(args, r.InteractionsPollDuration)...) + cmd.Args = append(cmd.Args, "-nc") + cmd.Env = r.buildEnv(env) + + output, err := cmd.CombinedOutput() + if debug && len(output) > 0 { + fmt.Println(string(output)) + } + + results := []string{} + + for line := range strings.SplitSeq(strings.TrimSpace(conversion.String(output)), "\n") { + trimmed := strings.TrimSpace(line) + + switch { + case strings.HasPrefix(trimmed, "[ERR]"), strings.HasPrefix(trimmed, "[WRN]"), strings.HasPrefix(trimmed, "[FTL]"): + results = append(results, trimmed) + } + } + + return results, err +} + +func (r *Runner) ArgsResultsWithEnv(debug bool, env []string, args ...string) ([]string, error) { + clone := r.Clone() + clone.InteractionsPollDuration = "5" + + return clone.BareResults(debug, env, args...) +} + +func (r *Runner) LoadedTemplates(binaryPath string, debug bool, args []string) (string, error) { + cmd := r.command(binaryPath, append(append([]string{}, args...), r.ExtraArgs...)...) + + cmd.Env = r.buildEnv(nil) + + if r.DisableAutoUpdate { + cmd.Args = append(cmd.Args, "-duc") + } + + if debug { + cmd.Args = append(cmd.Args, "-debug") + fmt.Println(cmd.String()) + } + + data, err := cmd.CombinedOutput() + if debug && len(data) > 0 { + fmt.Println(string(data)) + } + + if err != nil { + return "", err + } + + matches := templateLoaded.FindAllStringSubmatch(string(data), -1) + if len(matches) == 0 { + return "", errors.New("no matches found") + } + + return matches[0][1], nil +} + +func (r *Runner) CombinedOutput(debug bool, args []string) (string, error) { + builtArgs := r.buildArgs(args, r.InteractionsPollDuration) + cmd := r.command("", builtArgs...) + + cmd.Env = r.buildEnv(nil) + + if debug { + cmd.Args = append(cmd.Args, "-debug") + fmt.Println(cmd.String()) + } + + data, err := cmd.CombinedOutput() + if debug && len(data) > 0 { + fmt.Println(string(data)) + } + + if err != nil { + return "", err + } + + return string(data), nil +} + +// RunNucleiTemplateAndGetResults returns a list of results for a template. +func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return defaultRunner.TemplateResults(template, url, debug, extra...) +} + +// RunNucleiWorkflowAndGetResults returns a list of results for a workflow. +func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return defaultRunner.WorkflowResults(template, url, debug, extra...) +} + +func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { + if isTemplate { + return defaultRunner.TemplateResults(template, url, debug, extra...) + } else { + return defaultRunner.WorkflowResults(template, url, debug, extra...) + } +} + +func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ([]string, error) { + return defaultRunner.BareResults(debug, env, extra...) +} + +// RunNucleiWithArgsAndGetResults returns result,and runtime errors. +func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) { + return defaultRunner.ArgsResults(debug, args...) +} + +// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL). +func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { + return defaultRunner.ArgsErrors(debug, env, extra...) +} + +// RunNucleiArgsWithEnvAndGetResults returns a list of results in nuclei output (ERR,WRN,FTL). +func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string) ([]string, error) { + return defaultRunner.ArgsResultsWithEnv(debug, env, extra...) +} + +var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) + +// RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template. +func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) { + return defaultRunner.LoadedTemplates(nucleiBinary, debug, args) +} + +func RunNucleiBinaryAndGetCombinedOutput(debug bool, args []string) (string, error) { + return defaultRunner.CombinedOutput(debug, args) +} + +// TestCase is a single integration test case. +type TestCase interface { + // Execute executes a test case and returns any errors if occurred + Execute(filePath string) error +} + +// TCPServer creates a new tcp server that returns a response. +type TCPServer struct { + URL string + listener net.Listener +} + +// keys taken from https://pascal.bach.ch/2015/12/17/from-tcp-to-tls-in-go/ +const serverKey = `-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MIGkAgEBBDBJazGwuqgOLsCMr7P56w26JBEHQokiuAy2iCQfCnmOWm7S9FveQ/DP +qB69zvUPs26gBwYFK4EEACKhZANiAARehvy96ygCAsJ6iQvthzl/Nvq4P3c4MGyx +UMLMe0L10OCxeCl5ZY2CuFf8UnBgV1u414U4+yjIrS57w1/3utBKC9TVRGj+Vcls +2NZ4+8Jh6/M/Jf/Mpd8QyIy0WesEUM4= +-----END EC PRIVATE KEY----- +` + +const serverCert = `-----BEGIN CERTIFICATE----- +MIICJDCCAakCCQDFa0/D9jJw6DAKBggqhkjOPQQDAjB7MQswCQYDVQQGEwJVUzEP +MA0GA1UECAwGcGRsYW5kMQ8wDQYDVQQHDAZwZGNpdHkxCzAJBgNVBAoMAnBkMQsw +CQYDVQQLDAJwZDELMAkGA1UEAwwCcGQxIzAhBgkqhkiG9w0BCQEWFGFueXRoaW5n +QGFueXRoaW5nLnBkMB4XDTIyMDEyNzIyMDUwNFoXDTMyMDEyNTIyMDUwNFowezEL +MAkGA1UEBhMCVVMxDzANBgNVBAgMBnBkbGFuZDEPMA0GA1UEBwwGcGRjaXR5MQsw +CQYDVQQKDAJwZDELMAkGA1UECwwCcGQxCzAJBgNVBAMMAnBkMSMwIQYJKoZIhvcN +AQkBFhRhbnl0aGluZ0Bhbnl0aGluZy5wZDB2MBAGByqGSM49AgEGBSuBBAAiA2IA +BF6G/L3rKAICwnqJC+2HOX82+rg/dzgwbLFQwsx7QvXQ4LF4KXlljYK4V/xScGBX +W7jXhTj7KMitLnvDX/e60EoL1NVEaP5VyWzY1nj7wmHr8z8l/8yl3xDIjLRZ6wRQ +zjAKBggqhkjOPQQDAgNpADBmAjEAgxGPbjRlhz+1Scmr6RU9VbzVJWN8KCsTTpx7 +pqfmKpJ29UYReZN+fm/6fc5vkv1rAjEAkTuTf8ARSn1UiKlCTTDQVtCoRcMVLQQp +TCxxGzcAlUAAJE6+SJpY7fPRe+n2EvPS +-----END CERTIFICATE----- +` + +// NewTCPServer creates a new TCP server from a handler. +func NewTCPServer(tlsConfig *tls.Config, port int, handler func(conn net.Conn)) *TCPServer { + server := &TCPServer{} + + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + panic(err) + } + + server.URL = l.Addr().String() + server.listener = l + + if tlsConfig != nil { + cer, err := tls.X509KeyPair([]byte(serverCert), []byte(serverKey)) + if err != nil { + panic(err) + } + + tlsConfig.Certificates = []tls.Certificate{cer} + } + + go func() { + for { + // Listen for an incoming connection. + conn, err := l.Accept() + if err != nil { + continue + } + // Handle connections in a new goroutine. + if tlsConfig != nil { + connTls := tls.Server(conn, tlsConfig) + go handler(connTls) + } else { + go handler(conn) + } + } + }() + + return server +} + +// Close closes the TCP server. +func (s *TCPServer) Close() { + _ = s.listener.Close() +} + +// NewWebsocketServer creates a new websocket server from a handler. +func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { + handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { + w.WriteHeader(http.StatusBadRequest) + + return + } + + conn, _, _, err := ws.UpgradeHTTP(r, w) + if err != nil { + return + } + + go func() { + defer func() { + _ = conn.Close() + }() + + handler(conn) + }() + }) + + if path != "" { + router := httprouter.New() + router.HandlerFunc("*", "/test", handlerFunc) + + return httptest.NewServer(router) + } + + return httptest.NewServer(handlerFunc) +} diff --git a/pkg/testutils/testutils.go b/internal/tests/testutils/testutils.go similarity index 93% rename from pkg/testutils/testutils.go rename to internal/tests/testutils/testutils.go index 49b79e642..18092915f 100644 --- a/pkg/testutils/testutils.go +++ b/internal/tests/testutils/testutils.go @@ -27,13 +27,13 @@ import ( unitutils "github.com/projectdiscovery/utils/unit" ) -// Init initializes the protocols and their configurations +// Init initializes the protocols and their configurations. func Init(options *types.Options) { _ = protocolstate.Init(options) _ = protocolinit.Init(options) } -// Cleanup cleans up the protocols and their configurations +// Cleanup cleans up the protocols and their configurations. func Cleanup(options *types.Options) { protocolstate.Close(options.ExecutionId) } @@ -90,7 +90,7 @@ type TemplateInfo struct { Path string } -// NewMockExecuterOptions creates a new mock executeroptions struct +// NewMockExecuterOptions creates a new mock executeroptions struct. func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts := &protocols.ExecutorOptions{ @@ -130,15 +130,15 @@ type MockOutputWriter struct { WriteCallback func(o *output.ResultEvent) } -// NewMockOutputWriter creates a new mock output writer +// NewMockOutputWriter creates a new mock output writer. func NewMockOutputWriter(omomitTemplate bool) *MockOutputWriter { return &MockOutputWriter{aurora: aurora.NewAurora(false), omitTemplate: omomitTemplate} } -// Close closes the output writer interface +// Close closes the output writer interface. func (m *MockOutputWriter) Close() {} -// Colorizer returns the colorizer instance for writer +// Colorizer returns the colorizer instance for writer. func (m *MockOutputWriter) Colorizer() aurora.Aurora { return m.aurora } @@ -152,10 +152,11 @@ func (m *MockOutputWriter) Write(result *output.ResultEvent) error { if m.WriteCallback != nil { m.WriteCallback(result) } + return nil } -// Request writes a log the requests trace log +// Request writes a log the requests trace log. func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { if m.RequestCallback != nil { m.RequestCallback(templateID, url, requestType, err) @@ -167,32 +168,40 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven // if failure event has more than one result, write them all if len(wrappedEvent.Results) > 0 { errs := []error{} + for _, result := range wrappedEvent.Results { result.MatcherStatus = false // just in case - if err := m.Write(result); err != nil { + err := m.Write(result) + if err != nil { errs = append(errs, err) } } + if len(errs) > 0 { return multierr.Combine(errs...) } + return nil } // create event event := wrappedEvent.InternalEvent templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]), types.ToString(event["template-verifier"])) + var templateInfo model.Info if ti, ok := event["template-info"].(model.Info); ok { templateInfo = ti } + fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"])) if types.ToString(event["ip"]) != "" { fields.Ip = types.ToString(event["ip"]) } + if types.ToString(event["path"]) != "" { fields.Path = types.ToString(event["path"]) } + data := &output.ResultEvent{ Template: templatePath, TemplateURL: templateURL, @@ -214,6 +223,7 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven TemplateEncoded: m.encodeTemplate(types.ToString(event["template-path"])), Error: types.ToString(event["error"]), } + return m.Write(data) } @@ -226,6 +236,7 @@ func (w *MockOutputWriter) encodeTemplate(templatePath string) string { if err == nil && !w.omitTemplate && len(data) <= maxTemplateFileSizeForEncoding && config.DefaultConfig.IsCustomTemplate(templatePath) { return base64.StdEncoding.EncodeToString(data) } + return "" } @@ -236,16 +247,16 @@ type MockProgressClient struct{} // Stop stops the progress recorder. func (m *MockProgressClient) Stop() {} -// Init inits the progress bar with initial details for scan +// Init inits the progress bar with initial details for scan. func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} -// AddToTotal adds a value to the total request count +// AddToTotal adds a value to the total request count. func (m *MockProgressClient) AddToTotal(delta int64) {} // IncrementRequests increments the requests counter by 1. func (m *MockProgressClient) IncrementRequests() {} -// SetRequests sets the counter by incrementing it with a delta +// SetRequests sets the counter by incrementing it with a delta. func (m *MockProgressClient) SetRequests(count uint64) {} // IncrementMatched increments the matched counter by 1. diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 83f3c6456..f56dedc4e 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -32,7 +32,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/types" nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/projectdiscovery/ratelimit" diff --git a/pkg/catalog/loader/loader_bench_test.go b/pkg/catalog/loader/loader_bench_test.go index 569a12dcb..09a20b2c8 100644 --- a/pkg/catalog/loader/loader_bench_test.go +++ b/pkg/catalog/loader/loader_bench_test.go @@ -11,7 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func BenchmarkStoreValidateTemplates(b *testing.B) { diff --git a/pkg/external/customtemplates/github_test.go b/pkg/external/customtemplates/github_test.go index 4e429c020..586fa3eee 100644 --- a/pkg/external/customtemplates/github_test.go +++ b/pkg/external/customtemplates/github_test.go @@ -10,7 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/utils" "github.com/stretchr/testify/require" ) diff --git a/pkg/input/formats/testdata/ytt/ytt-profile.yaml b/pkg/input/formats/testdata/ytt/ytt-profile.yaml index 01e794ad4..0eace9f68 100644 --- a/pkg/input/formats/testdata/ytt/ytt-profile.yaml +++ b/pkg/input/formats/testdata/ytt/ytt-profile.yaml @@ -1,7 +1,7 @@ list: pkg/input/formats/testdata/ytt/ginandjuice.ytt.yaml input-mode: yaml templates: - - integration_tests/fuzz/fuzz-body.yaml + - internal/tests/integration/testdata/fuzz/fuzz-body.yaml var: - debug=true - bar=bar diff --git a/pkg/protocols/code/code_test.go b/pkg/protocols/code/code_test.go index 675a1b6f7..82ac87b5d 100644 --- a/pkg/protocols/code/code_test.go +++ b/pkg/protocols/code/code_test.go @@ -12,7 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func newTestRequest(t *testing.T, engine []string, source string) (*Request, *testutils.TemplateInfo) { diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index 698c7f05e..015a40f5c 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -25,7 +25,7 @@ import ( httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/useragent" mapsutil "github.com/projectdiscovery/utils/maps" diff --git a/pkg/protocols/dns/dns_test.go b/pkg/protocols/dns/dns_test.go index bf2c3a9f7..b811109b4 100644 --- a/pkg/protocols/dns/dns_test.go +++ b/pkg/protocols/dns/dns_test.go @@ -7,7 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestDNSCompileMake(t *testing.T) { diff --git a/pkg/protocols/dns/operators_test.go b/pkg/protocols/dns/operators_test.go index 80d02908d..532f57bf6 100644 --- a/pkg/protocols/dns/operators_test.go +++ b/pkg/protocols/dns/operators_test.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/pkg/protocols/dns/request_test.go b/pkg/protocols/dns/request_test.go index 3d5a0396a..6e8fc476e 100644 --- a/pkg/protocols/dns/request_test.go +++ b/pkg/protocols/dns/request_test.go @@ -13,7 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestDNSExecuteWithResults(t *testing.T) { diff --git a/pkg/protocols/file/find_test.go b/pkg/protocols/file/find_test.go index 43894ae64..011467c84 100644 --- a/pkg/protocols/file/find_test.go +++ b/pkg/protocols/file/find_test.go @@ -9,7 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" permissionutil "github.com/projectdiscovery/utils/permission" ) diff --git a/pkg/protocols/file/operators_test.go b/pkg/protocols/file/operators_test.go index 72a0ca45b..6ea20faec 100644 --- a/pkg/protocols/file/operators_test.go +++ b/pkg/protocols/file/operators_test.go @@ -11,7 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func newMockOperator() operators.Operators { diff --git a/pkg/protocols/file/request_test.go b/pkg/protocols/file/request_test.go index 3f20de2ed..b1dfc0ee9 100644 --- a/pkg/protocols/file/request_test.go +++ b/pkg/protocols/file/request_test.go @@ -21,7 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" permissionutil "github.com/projectdiscovery/utils/permission" ) diff --git a/pkg/protocols/headless/engine/page_actions_test.go b/pkg/protocols/headless/engine/page_actions_test.go index bbd95de36..fc567f300 100644 --- a/pkg/protocols/headless/engine/page_actions_test.go +++ b/pkg/protocols/headless/engine/page_actions_test.go @@ -18,9 +18,9 @@ import ( "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testheadless" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils/testheadless" "github.com/projectdiscovery/nuclei/v3/pkg/types" envutil "github.com/projectdiscovery/utils/env" stringsutil "github.com/projectdiscovery/utils/strings" @@ -582,16 +582,17 @@ func TestActionWaitDialog(t *testing.T) { Nuclei Test Page - + + ` @@ -599,15 +600,16 @@ func TestActionWaitDialog(t *testing.T) { actions := []*Action{ { ActionType: ActionTypeHolder{ActionType: ActionNavigate}, - Data: map[string]string{"url": "{{BaseURL}}/?script=alert%281%29"}, + Data: map[string]string{"url": "{{BaseURL}}/?autoclick=true&delay=500"}, }, { ActionType: ActionTypeHolder{ActionType: ActionWaitDialog}, Name: "test", + Data: map[string]string{"max-duration": "2s"}, }, } - testHeadlessSimpleResponse(t, response, actions, 1*time.Second, func(page *Page, err error, out ActionData) { + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out ActionData) { require.Nil(t, err, "could not run page actions") test, ok := out["test"].(bool) @@ -620,11 +622,12 @@ func TestActionWaitDialog(t *testing.T) { actions := []*Action{ { ActionType: ActionTypeHolder{ActionType: ActionNavigate}, - Data: map[string]string{"url": "{{BaseURL}}/?script=foo"}, + Data: map[string]string{"url": "{{BaseURL}}"}, }, { ActionType: ActionTypeHolder{ActionType: ActionWaitDialog}, Name: "test", + Data: map[string]string{"max-duration": "1s"}, }, } diff --git a/pkg/protocols/http/build_request_test.go b/pkg/protocols/http/build_request_test.go index 07d4743aa..e45a4ea9f 100644 --- a/pkg/protocols/http/build_request_test.go +++ b/pkg/protocols/http/build_request_test.go @@ -12,7 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" urlutil "github.com/projectdiscovery/utils/url" ) diff --git a/pkg/protocols/http/http_test.go b/pkg/protocols/http/http_test.go index 396fd55ee..a2b922628 100644 --- a/pkg/protocols/http/http_test.go +++ b/pkg/protocols/http/http_test.go @@ -8,7 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestHTTPCompile(t *testing.T) { diff --git a/pkg/protocols/http/operators_test.go b/pkg/protocols/http/operators_test.go index deb5319d9..2de4c7ff6 100644 --- a/pkg/protocols/http/operators_test.go +++ b/pkg/protocols/http/operators_test.go @@ -13,7 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/pkg/protocols/http/request_annotations_test.go b/pkg/protocols/http/request_annotations_test.go index e8899d2f4..4e06333ea 100644 --- a/pkg/protocols/http/request_annotations_test.go +++ b/pkg/protocols/http/request_annotations_test.go @@ -9,7 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/retryablehttp-go" "github.com/stretchr/testify/require" ) diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go index dfc5041a2..8fbfa6440 100644 --- a/pkg/protocols/http/request_test.go +++ b/pkg/protocols/http/request_test.go @@ -20,7 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestHTTPExtractMultipleReuse(t *testing.T) { diff --git a/pkg/protocols/javascript/js_test.go b/pkg/protocols/javascript/js_test.go index 1fb09fb6c..6c6bb27ba 100644 --- a/pkg/protocols/javascript/js_test.go +++ b/pkg/protocols/javascript/js_test.go @@ -15,7 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" javascript "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/ratelimit" "github.com/stretchr/testify/require" ) diff --git a/pkg/protocols/network/network_test.go b/pkg/protocols/network/network_test.go index 83dbc7fc6..b26fd0c18 100644 --- a/pkg/protocols/network/network_test.go +++ b/pkg/protocols/network/network_test.go @@ -7,7 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestNetworkCompileMake(t *testing.T) { diff --git a/pkg/protocols/network/operators_test.go b/pkg/protocols/network/operators_test.go index 88bc94b4d..33f54a881 100644 --- a/pkg/protocols/network/operators_test.go +++ b/pkg/protocols/network/operators_test.go @@ -11,7 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/pkg/protocols/network/request_test.go b/pkg/protocols/network/request_test.go index 7ff0f4882..6d12b514a 100644 --- a/pkg/protocols/network/request_test.go +++ b/pkg/protocols/network/request_test.go @@ -18,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestNetworkExecuteWithResults(t *testing.T) { diff --git a/pkg/protocols/offlinehttp/find_test.go b/pkg/protocols/offlinehttp/find_test.go index acbfdf49b..5d38424f4 100644 --- a/pkg/protocols/offlinehttp/find_test.go +++ b/pkg/protocols/offlinehttp/find_test.go @@ -10,7 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/operators" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" permissionutil "github.com/projectdiscovery/utils/permission" ) diff --git a/pkg/protocols/offlinehttp/operators_test.go b/pkg/protocols/offlinehttp/operators_test.go index 4fb8c39f7..75fe75ed3 100644 --- a/pkg/protocols/offlinehttp/operators_test.go +++ b/pkg/protocols/offlinehttp/operators_test.go @@ -13,7 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/pkg/protocols/ssl/ssl_test.go b/pkg/protocols/ssl/ssl_test.go index 52a3ae507..b43aef622 100644 --- a/pkg/protocols/ssl/ssl_test.go +++ b/pkg/protocols/ssl/ssl_test.go @@ -10,7 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" ) func TestSSLProtocol(t *testing.T) { diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index c8e68b3e0..e40a4c42e 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -7,7 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/stretchr/testify/require" ) diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 81728d9c2..6369a0154 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -26,7 +26,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/nuclei/v3/pkg/workflows" "github.com/projectdiscovery/ratelimit" "github.com/stretchr/testify/require" diff --git a/pkg/templates/signer/testdata/ci-private-key.pem b/pkg/templates/signer/testdata/ci-private-key.pem new file mode 100644 index 000000000..b7a09b710 --- /dev/null +++ b/pkg/templates/signer/testdata/ci-private-key.pem @@ -0,0 +1,5 @@ +-----BEGIN PD NUCLEI USER PRIVATE KEY----- +MHcCAQEEIEywlBGZ94ARrBT+1fTu/Ii7HGfJc4y7kK4aGYvDMYm5oAoGCCqGSM49 +AwEHoUQDQgAEnyVUkFKJx92/8doQ//VAPCrzB4dqvNgwLRZPC/oAieVpNG8HDGNw +PJ7qB7ovIfGwDOW98vQwsRG4TmgFlZr0rQ== +-----END PD NUCLEI USER PRIVATE KEY----- \ No newline at end of file diff --git a/pkg/templates/signer/testdata/ci.crt b/pkg/templates/signer/testdata/ci.crt new file mode 100644 index 000000000..0aa66b1b2 --- /dev/null +++ b/pkg/templates/signer/testdata/ci.crt @@ -0,0 +1,9 @@ +-----BEGIN PD NUCLEI USER CERTIFICATE----- +MIIBPzCB56ADAgECAgRlHGgmMAoGCCqGSM49BAMCMA0xCzAJBgNVBAMTAkNJMB4X +DTIzMTAwMzE5MTQ0NloXDTI3MTAwMjE5MTQ0NlowDTELMAkGA1UEAxMCQ0kwWTAT +BgcqhkjOPQIBBggqhkjOPQMBBwNCAASfJVSQUonH3b/x2hD/9UA8KvMHh2q82DAt +Fk8L+gCJ5Wk0bwcMY3A8nuoHui8h8bAM5b3y9DCxEbhOaAWVmvStozUwMzAOBgNV +HQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAK +BggqhkjOPQQDAgNHADBEAiBgUdbAcSbDpkNNQscZog/pAuaRV4sk7fbOlTRcjZTL +qQIgdtvG1w7l9VAtk6gx+HJa3BP9IFhSfT+a3UCuJy2p2iA= +-----END PD NUCLEI USER CERTIFICATE----- \ No newline at end of file diff --git a/pkg/templates/signer/tmpl_signer_test.go b/pkg/templates/signer/tmpl_signer_test.go index 832dcb0e0..e89849087 100644 --- a/pkg/templates/signer/tmpl_signer_test.go +++ b/pkg/templates/signer/tmpl_signer_test.go @@ -11,8 +11,8 @@ import ( ) const ( - testCertFile = "../../../integration_tests/protocols/keys/ci.crt" - testKeyFile = "../../../integration_tests/protocols/keys/ci-private-key.pem" + testCertFile = "testdata/ci.crt" + testKeyFile = "testdata/ci-private-key.pem" ) type mockSignableTemplate struct { diff --git a/pkg/testutils/integration.go b/pkg/testutils/integration.go deleted file mode 100644 index d93e87011..000000000 --- a/pkg/testutils/integration.go +++ /dev/null @@ -1,355 +0,0 @@ -package testutils - -import ( - "crypto/tls" - "errors" - "fmt" - "net" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "regexp" - "strings" - - "github.com/gobwas/ws" - "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/utils/conversion" -) - -// ExtraArgs -var ( - ExtraDebugArgs = []string{} - ExtraEnvVars = []string{ - "DISABLE_CLOUD_UPLOAD_WRN=true", - "DISABLE_CLOUD_UPLOAD=true", - } -) - -// RunNucleiTemplateAndGetResults returns a list of results for a template -func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return RunNucleiAndGetResults(true, template, url, debug, extra...) -} - -// RunNucleiWorkflowAndGetResults returns a list of results for a workflow -func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return RunNucleiAndGetResults(false, template, url, debug, extra...) -} - -func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { - var templateOrWorkflowFlag string - if isTemplate { - templateOrWorkflowFlag = "-t" - } else { - templateOrWorkflowFlag = "-w" - } - - return RunNucleiBareArgsAndGetResults(debug, nil, append([]string{ - templateOrWorkflowFlag, - template, - "-target", - url, - }, extra...)...) -} - -func RunNucleiBareArgsAndGetResults(debug bool, env []string, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei") - extra = append(extra, ExtraDebugArgs...) - cmd.Args = append(cmd.Args, extra...) - cmd.Args = append(cmd.Args, "-duc") // disable auto updates - cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") - cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") - cmd.Args = append(cmd.Args, "-allow-local-file-access") - if env != nil { - cmd.Env = append(os.Environ(), env...) - } - cmd.Env = append(cmd.Env, ExtraEnvVars...) - if debug { - cmd.Args = append(cmd.Args, "-debug") - cmd.Stderr = os.Stderr - fmt.Println(cmd.String()) - } else { - cmd.Args = append(cmd.Args, "-silent") - } - output, err := cmd.Output() - var data string - if len(output) > 0 { - data = strings.TrimSpace(conversion.String(output)) - } - if debug { - fmt.Println(data) - } - var parts []string - items := strings.Split(data, "\n") - for _, i := range items { - if i != "" { - parts = append(parts, i) - } - } - - if (data == "" || len(parts) == 0) && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), data) - } - - return parts, nil -} - -// RunNucleiWithArgsAndGetResults returns result,and runtime errors -func RunNucleiWithArgsAndGetResults(debug bool, args ...string) ([]string, error) { - cmd := exec.Command("./nuclei", args...) - cmd.Env = append(cmd.Env, ExtraEnvVars...) - if debug { - cmd.Args = append(cmd.Args, "-debug") - cmd.Stderr = os.Stderr - fmt.Println(cmd.String()) - } else { - cmd.Args = append(cmd.Args, "-silent") - } - output, err := cmd.Output() - var data string - if len(output) > 0 { - data = strings.TrimSpace(conversion.String(output)) - } - if debug { - fmt.Println(data) - } - var parts []string - items := strings.Split(data, "\n") - for _, i := range items { - if i != "" { - parts = append(parts, i) - } - } - - if (data == "" || len(parts) == 0) && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), data) - } - return parts, nil -} - -// RunNucleiArgsAndGetErrors returns a list of errors in nuclei output (ERR,WRN,FTL) -func RunNucleiArgsAndGetErrors(debug bool, env []string, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei") - extra = append(extra, ExtraDebugArgs...) - cmd.Env = append(os.Environ(), env...) - cmd.Args = append(cmd.Args, extra...) - cmd.Args = append(cmd.Args, "-duc") // disable auto updates - cmd.Args = append(cmd.Args, "-interactions-poll-duration", "1") - cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") - cmd.Args = append(cmd.Args, "-allow-local-file-access") - cmd.Args = append(cmd.Args, "-nc") // disable color - cmd.Env = append(cmd.Env, ExtraEnvVars...) - dataOutput, err := cmd.CombinedOutput() - if debug { - fmt.Println(string(dataOutput)) - } - var data string - if len(dataOutput) > 0 { - data = strings.TrimSpace(conversion.String(dataOutput)) - } - results := []string{} - for _, v := range strings.Split(data, "\n") { - line := strings.TrimSpace(v) - switch { - case strings.HasPrefix(line, "[ERR]"): - results = append(results, line) - case strings.HasPrefix(line, "[WRN]"): - results = append(results, line) - case strings.HasPrefix(line, "[FTL]"): - results = append(results, line) - } - } - return results, err -} - -// RunNucleiArgsWithEnvAndGetResults returns a list of results in nuclei output (ERR,WRN,FTL) -func RunNucleiArgsWithEnvAndGetResults(debug bool, env []string, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei") - extra = append(extra, ExtraDebugArgs...) - cmd.Env = append(os.Environ(), env...) - cmd.Env = append(cmd.Env, ExtraEnvVars...) - cmd.Args = append(cmd.Args, extra...) - cmd.Args = append(cmd.Args, "-duc") // disable auto updates - cmd.Args = append(cmd.Args, "-interactions-poll-duration", "5") - cmd.Args = append(cmd.Args, "-interactions-cooldown-period", "10") - cmd.Args = append(cmd.Args, "-allow-local-file-access") - if debug { - cmd.Args = append(cmd.Args, "-debug") - cmd.Stderr = os.Stderr - fmt.Println(cmd.String()) - } else { - cmd.Args = append(cmd.Args, "-silent") - } - dataOutput, err := cmd.Output() - var data string - if len(dataOutput) > 0 { - data = strings.TrimSpace(conversion.String(dataOutput)) - } - if debug { - fmt.Println(data) - } - var parts []string - items := strings.Split(data, "\n") - for _, i := range items { - if i != "" { - parts = append(parts, i) - } - } - if (data == "" || len(parts) == 0) && err != nil { - return nil, fmt.Errorf("%v: %v", err.Error(), data) - } - return parts, nil -} - -var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) - -// RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template -func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) { - cmd := exec.Command(nucleiBinary, args...) - cmd.Env = append(cmd.Env, ExtraEnvVars...) - cmd.Args = append(cmd.Args, "-duc") // disable auto updates - if debug { - cmd.Args = append(cmd.Args, "-debug") - fmt.Println(cmd.String()) - } - data, err := cmd.CombinedOutput() - if debug { - fmt.Println(string(data)) - } - if err != nil { - return "", err - } - matches := templateLoaded.FindAllStringSubmatch(string(data), -1) - if len(matches) == 0 { - return "", errors.New("no matches found") - } - return matches[0][1], nil -} -func RunNucleiBinaryAndGetCombinedOutput(debug bool, args []string) (string, error) { - args = append(args, "-interactions-cooldown-period", "10", "-interactions-poll-duration", "1") - cmd := exec.Command("./nuclei", args...) - cmd.Env = append(cmd.Env, ExtraEnvVars...) - if debug { - cmd.Args = append(cmd.Args, "-debug") - fmt.Println(cmd.String()) - } - data, err := cmd.CombinedOutput() - if debug { - fmt.Println(string(data)) - } - if err != nil { - return "", err - } - return string(data), nil -} - -// TestCase is a single integration test case -type TestCase interface { - // Execute executes a test case and returns any errors if occurred - Execute(filePath string) error -} - -// TCPServer creates a new tcp server that returns a response -type TCPServer struct { - URL string - listener net.Listener -} - -// keys taken from https://pascal.bach.ch/2015/12/17/from-tcp-to-tls-in-go/ -const serverKey = `-----BEGIN EC PARAMETERS----- -BgUrgQQAIg== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MIGkAgEBBDBJazGwuqgOLsCMr7P56w26JBEHQokiuAy2iCQfCnmOWm7S9FveQ/DP -qB69zvUPs26gBwYFK4EEACKhZANiAARehvy96ygCAsJ6iQvthzl/Nvq4P3c4MGyx -UMLMe0L10OCxeCl5ZY2CuFf8UnBgV1u414U4+yjIrS57w1/3utBKC9TVRGj+Vcls -2NZ4+8Jh6/M/Jf/Mpd8QyIy0WesEUM4= ------END EC PRIVATE KEY----- -` - -const serverCert = `-----BEGIN CERTIFICATE----- -MIICJDCCAakCCQDFa0/D9jJw6DAKBggqhkjOPQQDAjB7MQswCQYDVQQGEwJVUzEP -MA0GA1UECAwGcGRsYW5kMQ8wDQYDVQQHDAZwZGNpdHkxCzAJBgNVBAoMAnBkMQsw -CQYDVQQLDAJwZDELMAkGA1UEAwwCcGQxIzAhBgkqhkiG9w0BCQEWFGFueXRoaW5n -QGFueXRoaW5nLnBkMB4XDTIyMDEyNzIyMDUwNFoXDTMyMDEyNTIyMDUwNFowezEL -MAkGA1UEBhMCVVMxDzANBgNVBAgMBnBkbGFuZDEPMA0GA1UEBwwGcGRjaXR5MQsw -CQYDVQQKDAJwZDELMAkGA1UECwwCcGQxCzAJBgNVBAMMAnBkMSMwIQYJKoZIhvcN -AQkBFhRhbnl0aGluZ0Bhbnl0aGluZy5wZDB2MBAGByqGSM49AgEGBSuBBAAiA2IA -BF6G/L3rKAICwnqJC+2HOX82+rg/dzgwbLFQwsx7QvXQ4LF4KXlljYK4V/xScGBX -W7jXhTj7KMitLnvDX/e60EoL1NVEaP5VyWzY1nj7wmHr8z8l/8yl3xDIjLRZ6wRQ -zjAKBggqhkjOPQQDAgNpADBmAjEAgxGPbjRlhz+1Scmr6RU9VbzVJWN8KCsTTpx7 -pqfmKpJ29UYReZN+fm/6fc5vkv1rAjEAkTuTf8ARSn1UiKlCTTDQVtCoRcMVLQQp -TCxxGzcAlUAAJE6+SJpY7fPRe+n2EvPS ------END CERTIFICATE----- -` - -// NewTCPServer creates a new TCP server from a handler -func NewTCPServer(tlsConfig *tls.Config, port int, handler func(conn net.Conn)) *TCPServer { - server := &TCPServer{} - - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - panic(err) - } - server.URL = l.Addr().String() - server.listener = l - - if tlsConfig != nil { - cer, err := tls.X509KeyPair([]byte(serverCert), []byte(serverKey)) - if err != nil { - panic(err) - } - tlsConfig.Certificates = []tls.Certificate{cer} - } - - go func() { - for { - // Listen for an incoming connection. - conn, err := l.Accept() - if err != nil { - continue - } - // Handle connections in a new goroutine. - if tlsConfig != nil { - connTls := tls.Server(conn, tlsConfig) - go handler(connTls) - } else { - go handler(conn) - } - } - }() - return server -} - -// Close closes the TCP server -func (s *TCPServer) Close() { - _ = s.listener.Close() -} - -// NewWebsocketServer creates a new websocket server from a handler -func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { - handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { - w.WriteHeader(http.StatusBadRequest) - return - } - conn, _, _, err := ws.UpgradeHTTP(r, w) - if err != nil { - return - } - go func() { - defer func() { - _ = conn.Close() - }() - - handler(conn) - }() - }) - - if path != "" { - router := httprouter.New() - router.HandlerFunc("*", "/test", handlerFunc) - return httptest.NewServer(router) - } - return httptest.NewServer(handlerFunc) -} diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index 2a3c43d7d..9786d05cd 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -18,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/ratelimit" "github.com/stretchr/testify/require" ) diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 6be10dc77..8adda42f8 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -16,7 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/templates" - "github.com/projectdiscovery/nuclei/v3/pkg/testutils" + "github.com/projectdiscovery/nuclei/v3/internal/tests/testutils" "github.com/projectdiscovery/ratelimit" "github.com/stretchr/testify/require" ) diff --git a/static/regression-cycle.md b/static/regression-cycle.md deleted file mode 100644 index 8b1378917..000000000 --- a/static/regression-cycle.md +++ /dev/null @@ -1 +0,0 @@ -