admin/projectdiscovery-nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei synced 2026-06-08 16:50:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: native tests (#7307)

Browse Source 
Signed-off-by: Dwi Siswanto <git@dw1.io>
This commit is contained in:
Dwi Siswanto
2026-04-15 05:27:07 +07:00
committed by GitHub
parent 0e67be77bc
commit f893b6c0cb
330 changed files with 3520 additions and 3137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/flamegraph.yaml
+1 -2
View File
@@ -14,8 +14,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- run: make build
- name: "Setup environment (push)"
.github/workflows/generate-pgo.yaml
+1 -2
View File
@@ -20,8 +20,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- run: |
for i in {1..${{ env.TARGET_COUNT }}}; do
echo "${{ env.TARGET }}/-/?_=${i}" >> "${{ env.TARGET_LIST }}";
.github/workflows/perf-regression.yaml
+1 -2
View File
@@ -13,8 +13,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- run: make build-test
- run: ./bin/nuclei.test -test.run - -test.bench=. -test.benchmem ./cmd/nuclei/ | tee $BENCH_OUT
env:
.github/workflows/tests.yaml
+12 -17
View File
@@ -24,7 +24,6 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/golangci-lint/v2@v1
tests:
@@ -40,8 +39,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- uses: projectdiscovery/actions/free-disk-space@v1
with:
llvm: 'false'
@@ -56,7 +54,7 @@ jobs:
- run: make test
env:
PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
- run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
- run: go run -race . -l ${{ github.workspace }}/internal/tests/functional/testdata/targets.txt -id tech-detect,tls-version
if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099
working-directory: cmd/nuclei/
@@ -69,8 +67,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- name: "Simple"
run: go run .
working-directory: examples/simple/
@@ -94,15 +91,12 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/actions/setup/python@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- run: bash run.sh "${{ matrix.os }}"
- uses: projectdiscovery/actions/setup/go@v1
- run: make integration
env:
PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
timeout-minutes: 50
working-directory: integration_tests/
functional:
name: "Functional tests"
@@ -116,12 +110,13 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/nuclei@v1
- uses: projectdiscovery/actions/setup/python@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- run: bash run.sh
working-directory: cmd/functional-test/
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/nuclei-action@v3
with:
version: latest
install-only: true
- run: make functional
validate:
name: "Template validate"
@@ -132,7 +127,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/cache/go-rod-browser@v1
- uses: projectdiscovery/nuclei-action/cache@v3
- run: make template-validate
codeql:
.gitignore
+1 -2
View File
@@ -29,8 +29,7 @@
/scrapefunc
/scrapefuncs
/tsgen
/integration_tests/integration-test
/integration_tests/nuclei
*.test
# Templates
/*.yaml
.run/DSLFunctionsIT.run.xml
-15
View File
@@ -1,15 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="DSLFunctionsIT" type="GoApplicationRunConfiguration" factoryName="Go Application">
<module name="nuclei" />
<working_directory value="$PROJECT_DIR$/integration_tests" />
<envs>
<env name="DEBUG" value="true" />
<env name="TESTS" value="http/dsl-functions.yaml" />
</envs>
<kind value="PACKAGE" />
<package value="github.com/projectdiscovery/nuclei/v3/cmd/integration-test" />
<directory value="$PROJECT_DIR$" />
<filePath value="$PROJECT_DIR$/cmd/integration-test/integration-test.go" />
<method v="2" />
</configuration>
</component>
.run/IntegrationTests.run.xml
-19
View File
@@ -1,19 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="IntegrationTests" type="ShConfigurationType">
<option name="SCRIPT_TEXT" value="" />
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
<option name="SCRIPT_PATH" value="$PROJECT_DIR$/integration_tests/run.sh" />
<option name="SCRIPT_OPTIONS" value="" />
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/integration_tests/" />
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
<option name="INTERPRETER_PATH" value="/bin/zsh" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="EXECUTE_IN_TERMINAL" value="true" />
<option name="EXECUTE_SCRIPT_FILE" value="true" />
<envs>
<env name="DEBUG" value="true" />
</envs>
<method v="2" />
</configuration>
</component>
.run/RegressionTests.run.xml
-19
View File
@@ -1,19 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="RegressionTests" type="ShConfigurationType">
<option name="SCRIPT_TEXT" value="" />
<option name="INDEPENDENT_SCRIPT_PATH" value="true" />
<option name="SCRIPT_PATH" value="$PROJECT_DIR$/cmd/functional-test/run.sh" />
<option name="SCRIPT_OPTIONS" value="" />
<option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
<option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/cmd/functional-test/" />
<option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
<option name="INTERPRETER_PATH" value="/bin/zsh" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="EXECUTE_IN_TERMINAL" value="true" />
<option name="EXECUTE_SCRIPT_FILE" value="true" />
<envs>
<env name="DEBUG" value="true" />
</envs>
<method v="2" />
</configuration>
</component>
CLAUDE.md
+5 -4
View File
@@ -11,8 +11,8 @@ Nuclei is a modern, high-performance vulnerability scanner built in Go that leve
### Building and Testing
- `make build` - Build the main nuclei binary to ./bin/nuclei
- `make test` - Run unit tests with race detection
- `make integration` - Run integration tests (builds and runs test suite)
- `make functional` - Run functional tests
- `make integration` - Run the native integration suite via `go test -tags=integration ./internal/tests/integration`
- `make functional` - CI-only functional suite entry point
- `make vet` - Run go vet for code analysis
- `make tidy` - Clean up go modules
@@ -29,7 +29,7 @@ Nuclei is a modern, high-performance vulnerability scanner built in Go that leve
### Testing Specific Components
- Run single test: `go test -v ./pkg/path/to/package -run TestName`
- Integration tests are in `integration_tests/` and can be run via `make integration`
- Integration tests are in `internal/tests/integration/` and run via `go test -tags=integration ./internal/tests/integration`
## Architecture Overview
@@ -77,7 +77,8 @@ Each protocol (HTTP, DNS, Network, etc.) implements:
## Key Directories
- **lib/** - SDK for embedding nuclei as a library
- **examples/** - Usage examples for different scenarios
- **integration_tests/** - Integration test suite with protocol-specific tests
- **internal/tests/integration/** - Native integration suite and owned testdata for harness coverage
- **internal/tests/functional/** - CI-only native functional comparison suite and testcase assets
- **pkg/fuzz/** - Fuzzing engine and DAST capabilities
- **pkg/input/** - Input processing for various formats (Burp, OpenAPI, etc.)
- **pkg/reporting/** - Result export and issue tracking integrations
DESIGN.md
+1 -1
View File
@@ -454,7 +454,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
}
```
That's it, you've added a new protocol to Nuclei. The next good step would be to write integration tests which are described in `integration-tests` and `cmd/integration-tests` directories.
That's it, you've added a new protocol to Nuclei. The next good step would be to add native integration coverage under `internal/tests/integration` and run it with `go test -tags=integration ./internal/tests/integration`.
## Profiling and Tracing
Makefile
+23 -9
View File
@@ -2,6 +2,7 @@
GOCMD := go
GOBUILD := $(GOCMD) build
GOBUILD_OUTPUT :=
GOBUILD_OUTPUT_EXT :=
GOBUILD_PACKAGES :=
GOBUILD_ADDITIONAL_ARGS :=
GOMOD := $(GOCMD) mod
@@ -16,22 +17,26 @@ ifneq ($(shell go env GOOS),darwin)
LDFLAGS += -extldflags "-static"
endif
ifeq ($(shell go env GOOS),windows)
GOBUILD_OUTPUT_EXT := .exe
endif
.PHONY: all build build-stats clean devtools-all devtools-bindgen devtools-scrapefuncs fuzz fuzz-ci fuzz-tools
.PHONY: devtools-tsgen docs docgen dsl-docs functional go-build lint lint-strict fuzzplayground syntax-docs
.PHONY: integration jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint
.PHONY: integration integration-debug jsupdate-all jsupdate-bindgen jsupdate-tsgen memogen scan-charts test test-with-lint
.PHONY: tidy ts verify download vet template-validate build-fuzz discover-fuzz-packages
all: build
clean:
rm -f '${GOBUILD_OUTPUT}' 2>/dev/null
rm -f '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' 2>/dev/null
go-build: clean
go-build:
CGO_ENABLED=0 $(GOBUILD) -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
-o '${GOBUILD_OUTPUT}' $(GOBUILD_PACKAGES)
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' $(GOBUILD_PACKAGES)
build: GOFLAGS = -v -pgo=auto
build: GOFLAGS = -pgo=auto
build: GOBUILD_OUTPUT = ./bin/nuclei
build: GOBUILD_PACKAGES = cmd/nuclei/main.go
build: go-build
@@ -42,7 +47,7 @@ build-test: GOBUILD_PACKAGES = ./cmd/nuclei/
build-test: clean
build-test:
CGO_ENABLED=0 $(GOCMD) test -c -trimpath $(GOFLAGS) -ldflags '${LDFLAGS}' $(GOBUILD_ADDITIONAL_ARGS) \
-o '${GOBUILD_OUTPUT}' ${GOBUILD_PACKAGES}
-o '${GOBUILD_OUTPUT}${GOBUILD_OUTPUT_EXT}' ${GOBUILD_PACKAGES}
build-stats: GOBUILD_OUTPUT = ./bin/nuclei-stats
build-stats: GOBUILD_PACKAGES = cmd/nuclei/main.go
@@ -77,15 +82,24 @@ syntax-docs: docgen
syntax-docs:
./bin/docgen SYNTAX-REFERENCE.md nuclei-jsonschema.json
test: GOFLAGS = -race -v -timeout 30m -count 1
test: GOFLAGS = -race -v -timeout 1h -count 1
test:
$(GOTEST) $(GOFLAGS) ./...
integration:
cd integration_tests; bash run.sh
$(GOTEST) -tags=integration -timeout 1h ./internal/tests/integration
functional:
cd cmd/functional-test; bash run.sh
integration-debug:
$(GOTEST) -tags=integration ./internal/tests/integration -v $(GO_TEST_ARGS) -args $(INTEGRATION_ARGS)
functional: build
@release_binary="$$(command -v nuclei.exe 2>/dev/null || command -v nuclei 2>/dev/null)"; \
if [ -z "$$release_binary" ]; then \
echo "release nuclei binary not found on PATH"; \
exit 1; \
fi; \
RELEASE_BINARY="$$release_binary" DEV_BINARY="$(PWD)/bin/nuclei" \
$(GOTEST) -tags=functional -timeout 1h ./internal/tests/functional
tidy:
$(GOMOD) tidy
_typos.toml
+2 -2
View File
@@ -9,13 +9,13 @@ extend-exclude = [
"README_PT-BR.md",
"README_TR.md",
# Test fixtures and data files
"integration_tests/",
"internal/tests/integration/testdata/",
"pkg/input/formats/testdata/",
"pkg/output/stats/waf/",
# Deserialization test data
"pkg/protocols/common/helpers/deserialization/testdata/",
# Certificate/encoded data in test utilities
"pkg/testutils/integration.go",
"internal/tests/testutils/integration.go",
# Vendor directory
"vendor/",
# Go checksum file
cmd/functional-test/main.go
-127
View File
@@ -1,127 +0,0 @@
package main
import (
"bufio"
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/kitabisa/go-ci"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var (
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary")
devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary")
testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests")
)
func main() {
flag.Parse()
debug := os.Getenv("DEBUG") == "true" || os.Getenv("RUNNER_DEBUG") == "1"
if err, errored := runFunctionalTests(debug); err != nil {
log.Fatalf("Could not run functional tests: %s\n", err)
} else if errored {
os.Exit(1)
}
}
func runFunctionalTests(debug bool) (error, bool) {
file, err := os.Open(*testcases)
if err != nil {
return errors.Wrap(err, "could not open test cases"), true
}
defer func() {
_ = file.Close()
}()
errored, failedTestCases := runTestCases(file, debug)
if ci.IsCI() {
fmt.Println("::group::Failed tests with debug")
for _, failedTestCase := range failedTestCases {
_ = runTestCase(failedTestCase, true)
}
fmt.Println("::endgroup::")
}
return nil, errored
}
func runTestCases(file *os.File, debug bool) (bool, []string) {
errored := false
var failedTestCases []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
testCase := strings.TrimSpace(scanner.Text())
if testCase == "" {
continue
}
// skip comments
if strings.HasPrefix(testCase, "#") {
continue
}
if runTestCase(testCase, debug) {
errored = true
failedTestCases = append(failedTestCases, testCase)
}
}
return errored, failedTestCases
}
func runTestCase(testCase string, debug bool) bool {
if err := runIndividualTestCase(testCase, debug); err != nil {
fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testCase, err)
return true
} else {
fmt.Printf("%s Test \"%s\" passed!\n", success, testCase)
}
return false
}
func runIndividualTestCase(testcase string, debug bool) error {
quoted := false
// split upon unquoted spaces
parts := strings.FieldsFunc(testcase, func(r rune) bool {
if r == '"' {
quoted = !quoted
}
return !quoted && r == ' '
})
// Quoted strings containing spaces are expressions and must have trailing \" removed
for index, part := range parts {
if strings.Contains(part, " ") {
parts[index] = strings.Trim(part, "\"")
}
}
var finalArgs []string
if len(parts) > 1 {
finalArgs = parts[1:]
}
mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, debug, finalArgs)
if err != nil {
return errors.Wrap(err, "could not run nuclei main test")
}
devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, debug, finalArgs)
if err != nil {
return errors.Wrap(err, "could not run nuclei dev test")
}
if mainOutput == devOutput {
return nil
}
return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
}
cmd/functional-test/run.sh
-43
View File
@@ -1,43 +0,0 @@
#!/bin/bash
if [ "${RUNNER_OS}" == "Windows" ]; then
EXT=".exe"
elif [ "${RUNNER_OS}" == "macOS" ]; then
if [ "${CI}" == "true" ]; then
sudo sysctl -w kern.maxfiles{,perproc}=524288
sudo launchctl limit maxfiles 65536 524288
fi
ORIGINAL_ULIMIT="$(ulimit -n)"
ulimit -n 65536 || true
fi
mkdir -p .nuclei-config/nuclei/
touch .nuclei-config/nuclei/.nuclei-ignore
echo "::group::Building functional-test binary"
go build -o "functional-test${EXT}"
echo "::endgroup::"
echo "::group::Building Nuclei binary from current branch"
go build -o "nuclei-dev${EXT}" ../nuclei
echo "::endgroup::"
echo "::group::Building latest release of nuclei"
go build -o "nuclei${EXT}" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei
echo "::endgroup::"
echo "::group::Installing nuclei templates"
eval "./nuclei-dev${EXT} -update-templates"
echo "::endgroup::"
echo "::group::Validating templates"
eval "./nuclei-dev${EXT} -validate"
echo "::endgroup::"
echo "Starting Nuclei functional test"
eval "./functional-test${EXT} -main ./nuclei${EXT} -dev ./nuclei-dev${EXT} -testcases testcases.txt"
if [ "${RUNNER_OS}" == "macOS" ]; then
ulimit -n "${ORIGINAL_ULIMIT}" || true
fi
cmd/integration-test/custom-dir.go
-40
View File
@@ -1,40 +0,0 @@
package main
import (
"os"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
type customConfigDirTest struct{}
var customConfigDirTestCases = []TestCaseInfo{
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &customConfigDirTest{}},
}
// Execute executes a test case and returns an error if occurred
func (h *customConfigDirTest) Execute(filePath string) error {
customTempDirectory, err := os.MkdirTemp("", "")
if err != nil {
return err
}
defer func() {
_ = os.RemoveAll(customTempDirectory)
}()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDirectory}, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}
if len(results) == 0 {
return nil
}
files, err := os.ReadDir(customTempDirectory)
if err != nil {
return err
}
var fileNames []string
for _, file := range files {
fileNames = append(fileNames, file.Name())
}
return expectResultsCount(fileNames, 4)
}
cmd/integration-test/dns.go
-106
View File
@@ -1,106 +0,0 @@
package main
import (
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var dnsTestCases = []TestCaseInfo{
{Path: "protocols/dns/a.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/aaaa.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/cname.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/srv.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/ns.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/txt.yaml", TestCase: &dnsBasic{}},
{Path: "protocols/dns/ptr.yaml", TestCase: &dnsPtr{}},
{Path: "protocols/dns/caa.yaml", TestCase: &dnsCAA{}},
{Path: "protocols/dns/tlsa.yaml", TestCase: &dnsTLSA{}},
{Path: "protocols/dns/variables.yaml", TestCase: &dnsVariables{}},
{Path: "protocols/dns/payload.yaml", TestCase: &dnsPayload{}},
{Path: "protocols/dns/dsl-matcher-variable.yaml", TestCase: &dnsDSLMatcherVariable{}},
}
type dnsBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsBasic) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type dnsPtr struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsPtr) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "1.1.1.1", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type dnsCAA struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsCAA) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type dnsTLSA struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsTLSA) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
type dnsVariables struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsVariables) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type dnsPayload struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsPayload) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug)
if err != nil {
return err
}
if err := expectResultsCount(results, 3); err != nil {
return err
}
// override payload from CLI
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, "google.com", debug, "-var", "subdomain_wordlist=subdomains.txt")
if err != nil {
return err
}
return expectResultsCount(results, 4)
}
type dnsDSLMatcherVariable struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsDSLMatcherVariable) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
cmd/integration-test/file.go
-48
View File
@@ -1,48 +0,0 @@
package main
import (
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var fileTestcases = []TestCaseInfo{
{Path: "protocols/file/matcher-with-or.yaml", TestCase: &fileWithOrMatcher{}},
{Path: "protocols/file/matcher-with-and.yaml", TestCase: &fileWithAndMatcher{}},
{Path: "protocols/file/matcher-with-nested-and.yaml", TestCase: &fileWithAndMatcher{}},
{Path: "protocols/file/extract.yaml", TestCase: &fileWithExtractor{}},
}
type fileWithOrMatcher struct{}
// Execute executes a test case and returns an error if occurred
func (h *fileWithOrMatcher) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type fileWithAndMatcher struct{}
// Execute executes a test case and returns an error if occurred
func (h *fileWithAndMatcher) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type fileWithExtractor struct{}
// Execute executes a test case and returns an error if occurred
func (h *fileWithExtractor) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "protocols/file/data/", debug, "-file")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
cmd/integration-test/flow.go
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var flowTestcases = []TestCaseInfo{
{Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}},
{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
{Path: "flow/iterate-one-value-flow.yaml", TestCase: &iterateOneValueFlow{}},
{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
{Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}},
}
type conditionalFlow struct{}
func (t *conditionalFlow) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "cloud.projectdiscovery.io", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type conditionalFlowNegative struct{}
func (t *conditionalFlowNegative) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
type iterateValuesFlow struct{}
func (t *iterateValuesFlow) Execute(filePath string) error {
router := httprouter.New()
testemails := []string{
"secrets@scanme.sh",
"superadmin@scanme.sh",
}
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = fmt.Fprint(w, testemails)
})
router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
type iterateOneValueFlow struct{}
func (t *iterateOneValueFlow) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type dnsNsProbe struct{}
func (t *dnsNsProbe) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug)
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
func getBase64(input string) string {
return base64.StdEncoding.EncodeToString([]byte(input))
}
type flowHideMatcher struct{}
func (t *flowHideMatcher) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
// this matcher should not return any results
return expectResultsCount(results, 0)
}
cmd/integration-test/integration-test.go
-285
View File
@@ -1,285 +0,0 @@
package main
import (
"flag"
"fmt"
"os"
"regexp"
"runtime"
"slices"
"strings"
"github.com/kitabisa/go-ci"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground"
sliceutil "github.com/projectdiscovery/utils/slice"
)
type TestCaseInfo struct {
Path string
TestCase testutils.TestCase
DisableOn func() bool
}
var (
debug = isDebugMode()
customTests = os.Getenv("TESTS")
protocol = os.Getenv("PROTO")
success = aurora.Green("[✓]").String()
failed = aurora.Red("[✘]").String()
protocolTests = map[string][]TestCaseInfo{
"http": httpTestcases,
"interactsh": interactshTestCases,
"network": networkTestcases,
"dns": dnsTestCases,
"workflow": workflowTestcases,
"loader": loaderTestcases,
"profile-loader": profileLoaderTestcases,
"websocket": websocketTestCases,
"headless": headlessTestcases,
"whois": whoisTestCases,
"ssl": sslTestcases,
"library": libraryTestcases,
"templatesPath": templatesPathTestCases,
"templatesDir": templatesDirTestCases,
"env_vars": templatesDirEnvTestCases,
"file": fileTestcases,
"offlineHttp": offlineHttpTestcases,
"customConfigDir": customConfigDirTestCases,
"fuzzing": fuzzingTestCases,
"code": codeTestCases,
"multi": multiProtoTestcases,
"generic": genericTestcases,
"dsl": dslTestcases,
"flow": flowTestcases,
"javascript": jsTestcases,
"matcher-status": matcherStatusTestcases,
"exporters": exportersTestCases,
}
// flakyTests are run with a retry count of 3
flakyTests = map[string]bool{
"protocols/http/self-contained-file-input.yaml": true,
}
// For debug purposes
runProtocol = ""
runTemplate = ""
extraArgs = []string{}
interactshRetryCount = 3
)
func main() {
flag.StringVar(&runProtocol, "protocol", "", "run integration tests of given protocol")
flag.StringVar(&runTemplate, "template", "", "run integration test of given template")
flag.Parse()
// allows passing extra args to nuclei
eargs := os.Getenv("DebugExtraArgs")
if eargs != "" {
extraArgs = strings.Split(eargs, " ")
testutils.ExtraDebugArgs = extraArgs
}
if runProtocol != "" {
debugTests()
os.Exit(1)
}
// start fuzz playground server
server := fuzzplayground.GetPlaygroundServer()
defer func() {
fuzzplayground.Cleanup()
_ = server.Close()
}()
go func() {
if err := server.Start("localhost:8082"); err != nil {
if !strings.Contains(err.Error(), "Server closed") {
gologger.Fatal().Msgf("Could not start server: %s\n", err)
}
}
}()
customTestsList := normalizeSplit(customTests)
failedTestTemplatePaths := runTests(customTestsList)
if len(failedTestTemplatePaths) > 0 {
if ci.IsCI() {
// run failed tests again assuming they are flaky
// if they fail as well only then we assume that there is an actual issue
fmt.Println("::group::Running failed tests again")
failedTestTemplatePaths = runTests(failedTestTemplatePaths)
fmt.Println("::endgroup::")
if len(failedTestTemplatePaths) > 0 {
debug = true
fmt.Println("::group::Failed integration tests in debug mode")
_ = runTests(failedTestTemplatePaths)
fmt.Println("::endgroup::")
} else {
fmt.Println("::group::All tests passed")
fmt.Println("::endgroup::")
os.Exit(0)
}
}
os.Exit(1)
}
}
// isDebugMode checks if debug mode is enabled via any of the supported debug
// environment variables.
func isDebugMode() bool {
debugEnvVars := []string{
"DEBUG",
"ACTIONS_RUNNER_DEBUG", // GitHub Actions runner debug
// Add more debug environment variables here as needed
}
truthyValues := []string{"true", "1", "yes", "on", "enabled"}
for _, envVar := range debugEnvVars {
envValue := strings.ToLower(strings.TrimSpace(os.Getenv(envVar)))
if slices.Contains(truthyValues, envValue) {
return true
}
}
return false
}
// execute a testcase with retry and consider best of N
// intended for flaky tests like interactsh
func executeWithRetry(testCase testutils.TestCase, templatePath string, retryCount int) (string, error) {
var err error
for i := 0; i < retryCount; i++ {
err = testCase.Execute(templatePath)
if err == nil {
fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath)
return "", nil
}
}
_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed after %v attempts : %s\n", failed, templatePath, retryCount, err)
return templatePath, err
}
func debugTests() {
testCaseInfos := protocolTests[runProtocol]
for _, testCaseInfo := range testCaseInfos {
if (runTemplate != "" && !strings.Contains(testCaseInfo.Path, runTemplate)) ||
(testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn()) {
continue
}
if runProtocol == "interactsh" {
if _, err := executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount); err != nil {
fmt.Printf("\n%v", err.Error())
}
} else {
if _, err := execute(testCaseInfo.TestCase, testCaseInfo.Path); err != nil {
fmt.Printf("\n%v", err.Error())
}
}
}
}
func runTests(customTemplatePaths []string) []string {
var failedTestTemplatePaths []string
for proto, testCaseInfos := range protocolTests {
if protocol != "" {
if !strings.EqualFold(proto, protocol) {
continue
}
}
if len(customTemplatePaths) == 0 {
fmt.Printf("Running test cases for %q protocol\n", aurora.Blue(proto))
}
for _, testCaseInfo := range testCaseInfos {
if testCaseInfo.DisableOn != nil && testCaseInfo.DisableOn() {
fmt.Printf("skipping test case %v. disabled on %v.\n", aurora.Blue(testCaseInfo.Path), runtime.GOOS)
continue
}
if len(customTemplatePaths) == 0 || sliceutil.Contains(customTemplatePaths, testCaseInfo.Path) {
var failedTemplatePath string
var err error
if proto == "interactsh" || strings.Contains(testCaseInfo.Path, "interactsh") {
failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
} else if flakyTests[testCaseInfo.Path] {
failedTemplatePath, err = executeWithRetry(testCaseInfo.TestCase, testCaseInfo.Path, interactshRetryCount)
} else {
failedTemplatePath, err = execute(testCaseInfo.TestCase, testCaseInfo.Path)
}
if err != nil {
failedTestTemplatePaths = append(failedTestTemplatePaths, failedTemplatePath)
}
}
}
}
return failedTestTemplatePaths
}
func execute(testCase testutils.TestCase, templatePath string) (string, error) {
if err := testCase.Execute(templatePath); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, templatePath, err)
return templatePath, err
}
fmt.Printf("%s Test \"%s\" passed!\n", success, templatePath)
return "", nil
}
func expectResultsCount(results []string, expectedNumbers ...int) error {
results = filterLines(results)
match := sliceutil.Contains(expectedNumbers, len(results))
if !match {
return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected) \nResults:\n\t%s\n", len(results), expectedNumbers, strings.Join(results, "\n\t")) // nolint:all
}
return nil
}
func normalizeSplit(str string) []string {
return strings.FieldsFunc(str, func(r rune) bool {
return r == ','
})
}
// filterLines applies all filtering functions to the results
func filterLines(results []string) []string {
results = filterHeadlessLogs(results)
results = filterUnsignedTemplatesWarnings(results)
return results
}
// if chromium is not installed go-rod installs it in .cache directory
// this function filters out the logs from download and installation
func filterHeadlessLogs(results []string) []string {
// [launcher.Browser] 2021/09/23 15:24:05 [launcher] [info] Starting browser
filtered := []string{}
for _, result := range results {
if strings.Contains(result, "[launcher.Browser]") {
continue
}
filtered = append(filtered, result)
}
return filtered
}
// filterUnsignedTemplatesWarnings filters out warning messages about unsigned templates
func filterUnsignedTemplatesWarnings(results []string) []string {
filtered := []string{}
unsignedTemplatesRegex := regexp.MustCompile(`Loading \d+ unsigned templates for scan\. Use with caution\.`)
for _, result := range results {
if unsignedTemplatesRegex.MatchString(result) {
continue
}
filtered = append(filtered, result)
}
return filtered
}
cmd/integration-test/javascript.go
-502
View File
@@ -1,502 +0,0 @@
package main
import (
"log"
"time"
"github.com/ory/dockertest/v3"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
osutils "github.com/projectdiscovery/utils/os"
"go.uber.org/multierr"
)
var jsTestcases = []TestCaseInfo{
{Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
{Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/oracle-auth-test.yaml", TestCase: &javascriptOracleAuthTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}},
{Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/mysql-connect.yaml", TestCase: &javascriptMySQLConnect{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/javascript/multi-ports.yaml", TestCase: &javascriptMultiPortsSSH{}},
{Path: "protocols/javascript/no-port-args.yaml", TestCase: &javascriptNoPortArgs{}},
{Path: "protocols/javascript/telnet-auth-test.yaml", TestCase: &javascriptTelnetAuthTest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
}
var (
redisResource *dockertest.Resource
sshResource *dockertest.Resource
oracleResource *dockertest.Resource
vncResource *dockertest.Resource
telnetResource *dockertest.Resource
postgresResource *dockertest.Resource
mysqlResource *dockertest.Resource
rsyncResource *dockertest.Resource
pool *dockertest.Pool
defaultRetry = 3
)
type javascriptNetHttps struct{}
func (j *javascriptNetHttps) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptRedisPassBrute struct{}
func (j *javascriptRedisPassBrute) Execute(filePath string) error {
if redisResource == nil || pool == nil {
// skip test as redis is not running
return nil
}
tempPort := redisResource.GetPort("6379/tcp")
finalURL := "localhost:" + tempPort
defer purge(redisResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let ssh server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptSSHServerFingerprint struct{}
func (j *javascriptSSHServerFingerprint) Execute(filePath string) error {
if sshResource == nil || pool == nil {
// skip test as redis is not running
return nil
}
tempPort := sshResource.GetPort("2222/tcp")
finalURL := "localhost:" + tempPort
defer purge(sshResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let ssh server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptOracleAuthTest struct{}
func (j *javascriptOracleAuthTest) Execute(filePath string) error {
if oracleResource == nil || pool == nil {
// skip test as oracle is not running
return nil
}
tempPort := oracleResource.GetPort("1521/tcp")
finalURL := "localhost:" + tempPort
defer purge(oracleResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
// let oracle server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptVncPassBrute struct{}
func (j *javascriptVncPassBrute) Execute(filePath string) error {
if vncResource == nil || pool == nil {
// skip test as vnc is not running
return nil
}
tempPort := vncResource.GetPort("5900/tcp")
finalURL := "localhost:" + tempPort
defer purge(vncResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let ssh server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptPostgresPassBrute struct{}
func (j *javascriptPostgresPassBrute) Execute(filePath string) error {
if postgresResource == nil || pool == nil {
// skip test as postgres is not running
return nil
}
tempPort := postgresResource.GetPort("5432/tcp")
finalURL := "localhost:" + tempPort
defer purge(postgresResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let postgres server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptMySQLConnect struct{}
func (j *javascriptMySQLConnect) Execute(filePath string) error {
if mysqlResource == nil || pool == nil {
// skip test as mysql is not running
return nil
}
tempPort := mysqlResource.GetPort("3306/tcp")
finalURL := "localhost:" + tempPort
defer purge(mysqlResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let mysql server start
time.Sleep(5 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptMultiPortsSSH struct{}
func (j *javascriptMultiPortsSSH) Execute(filePath string) error {
// use scanme.sh as target to ensure we match on the 2nd default port 22
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptNoPortArgs struct{}
func (j *javascriptNoPortArgs) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "yo.dawg", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptRsyncTest struct{}
func (j *javascriptRsyncTest) Execute(filePath string) error {
if rsyncResource == nil || pool == nil {
// skip test as rsync is not running
return nil
}
tempPort := rsyncResource.GetPort("873/tcp")
finalURL := "localhost:" + tempPort
defer purge(rsyncResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let rsync server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
type javascriptTelnetAuthTest struct{}
func (j *javascriptTelnetAuthTest) Execute(filePath string) error {
if telnetResource == nil || pool == nil {
// skip test as telnet is not running
return nil
}
tempPort := telnetResource.GetPort("23/tcp")
finalURL := "localhost:" + tempPort
defer purge(telnetResource)
errs := []error{}
for i := 0; i < defaultRetry; i++ {
results := []string{}
var err error
_ = pool.Retry(func() error {
//let telnet server start
time.Sleep(3 * time.Second)
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, finalURL, debug)
return nil
})
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err == nil {
return nil
} else {
errs = append(errs, err)
}
}
return multierr.Combine(errs...)
}
// purge any given resource if it is not nil
func purge(resource *dockertest.Resource) {
if resource != nil && pool != nil {
containerName := resource.Container.Name
_ = pool.Client.StopContainer(resource.Container.ID, 0)
err := pool.Purge(resource)
if err != nil {
log.Printf("Could not purge resource: %s", err)
}
_ = pool.RemoveContainerByName(containerName)
}
}
func init() {
// uses a sensible default on windows (tcp/http) and linux/osx (socket)
pool, err := dockertest.NewPool("")
if err != nil {
log.Printf("something went wrong with dockertest: %s", err)
return
}
// uses pool to try to connect to Docker
err = pool.Client.Ping()
if err != nil {
log.Printf("Could not connect to Docker: %s", err)
}
// setup a temporary redis instance
redisResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "redis",
Tag: "latest",
Cmd: []string{"redis-server", "--requirepass", "iamadmin"},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start resource: %s", err)
return
}
// by default expire after 30 sec
if err := redisResource.Expire(30); err != nil {
log.Printf("Could not expire resource: %s", err)
}
// setup a temporary ssh server
sshResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "lscr.io/linuxserver/openssh-server",
Tag: "latest",
Env: []string{
"PUID=1000",
"PGID=1000",
"TZ=Etc/UTC",
"PASSWORD_ACCESS=true",
"USER_NAME=admin",
"USER_PASSWORD=admin",
},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start resource: %s", err)
return
}
// by default expire after 30 sec
if err := sshResource.Expire(30); err != nil {
log.Printf("Could not expire resource: %s", err)
}
// setup a temporary oracle instance
oracleResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "gvenzl/oracle-xe",
Tag: "latest",
Env: []string{
"ORACLE_PASSWORD=mysecret",
},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start Oracle resource: %s", err)
return
}
// by default expire after 30 sec
if err := oracleResource.Expire(30); err != nil {
log.Printf("Could not expire Oracle resource: %s", err)
}
// setup a temporary vnc server
vncResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "dorowu/ubuntu-desktop-lxde-vnc",
Tag: "latest",
Env: []string{
"VNC_PASSWORD=mysecret",
},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start resource: %s", err)
return
}
// by default expire after 30 sec
if err := vncResource.Expire(30); err != nil {
log.Printf("Could not expire resource: %s", err)
}
// setup a temporary postgres instance
postgresResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "postgres",
Tag: "latest",
Env: []string{
"POSTGRES_PASSWORD=postgres",
"POSTGRES_USER=postgres",
},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start postgres resource: %s", err)
return
}
// by default expire after 30 sec
if err := postgresResource.Expire(30); err != nil {
log.Printf("Could not expire postgres resource: %s", err)
}
// setup a temporary mysql instance
mysqlResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "mysql",
Tag: "latest",
Env: []string{
"MYSQL_ROOT_PASSWORD=secret",
},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start mysql resource: %s", err)
return
}
// by default expire after 30 sec
if err := mysqlResource.Expire(30); err != nil {
log.Printf("Could not expire mysql resource: %s", err)
}
// setup a temporary rsync server
rsyncResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "alpine",
Tag: "latest",
Cmd: []string{"sh", "-c", "apk add --no-cache rsync shadow && useradd -m rsyncuser && echo 'rsyncuser:mysecret' | chpasswd && echo 'rsyncuser:MySecret123' > /etc/rsyncd.secrets && chmod 600 /etc/rsyncd.secrets && echo -e '[data]\\n path = /data\\n comment = Local Rsync Share\\n read only = false\\n auth users = rsyncuser\\n secrets file = /etc/rsyncd.secrets' > /etc/rsyncd.conf && mkdir -p /data && exec rsync --daemon --no-detach --config=/etc/rsyncd.conf"},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start Rsync resource: %s", err)
return
}
// by default expire after 30 sec
if err := rsyncResource.Expire(30); err != nil {
log.Printf("Could not expire Rsync resource: %s", err)
}
// setup a temporary telnet server
// username: dev
// password: mysecret
telnetResource, err = pool.RunWithOptions(&dockertest.RunOptions{
Repository: "alpine",
Tag: "latest",
Cmd: []string{"sh", "-c", "apk add --no-cache busybox-extras shadow && useradd -m dev && echo 'dev:mysecret' | chpasswd && exec /usr/sbin/telnetd -F -p 23 -l /bin/login"},
Platform: "linux/amd64",
})
if err != nil {
log.Printf("Could not start Telnet resource: %s", err)
return
}
// by default expire after 30 sec
if err := telnetResource.Expire(30); err != nil {
log.Printf("Could not expire Telnet resource: %s", err)
}
}
cmd/integration-test/loader.go
-229
View File
@@ -1,229 +0,0 @@
package main
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/utils/errkit"
permissionutil "github.com/projectdiscovery/utils/permission"
)
var loaderTestcases = []TestCaseInfo{
{Path: "loader/template-list.yaml", TestCase: &remoteTemplateList{}},
{Path: "loader/workflow-list.yaml", TestCase: &remoteWorkflowList{}},
{Path: "loader/excluded-template.yaml", TestCase: &excludedTemplate{}},
{Path: "loader/nonexistent-template-list.yaml", TestCase: &nonExistentTemplateList{}},
{Path: "loader/nonexistent-workflow-list.yaml", TestCase: &nonExistentWorkflowList{}},
{Path: "loader/template-list-not-allowed.yaml", TestCase: &remoteTemplateListNotAllowed{}},
{Path: "loader/load-template-with-id", TestCase: &loadTemplateWithID{}},
}
type remoteTemplateList struct{}
// Execute executes a test case and returns an error if occurred
func (h *remoteTemplateList) Execute(templateList string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
_, _ = fmt.Fprintf(w, "This is test headers matcher text")
}
})
router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(templateList)
if err != nil {
w.WriteHeader(500)
}
_, err = w.Write(file)
if err != nil {
w.WriteHeader(500)
}
})
ts := httptest.NewServer(router)
defer ts.Close()
configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
if err != nil {
return err
}
defer func() {
_ = os.Remove("test-config.yaml")
}()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list", "-config", "test-config.yaml")
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
type excludedTemplate struct{}
// Execute executes a test case and returns an error if occurred
func (h *excludedTemplate) Execute(templateList string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
_, _ = fmt.Fprintf(w, "This is test headers matcher text")
}
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-t", templateList, "-include-templates", templateList)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type remoteTemplateListNotAllowed struct{}
// Execute executes a test case and returns an error if occurred
func (h *remoteTemplateListNotAllowed) Execute(templateList string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
_, _ = fmt.Fprintf(w, "This is test headers matcher text")
}
})
router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(templateList)
if err != nil {
w.WriteHeader(500)
}
_, err = w.Write(file)
if err != nil {
w.WriteHeader(500)
}
})
ts := httptest.NewServer(router)
defer ts.Close()
_, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/template_list")
if err == nil {
return fmt.Errorf("expected error for not allowed remote template list url")
}
return nil
}
type remoteWorkflowList struct{}
// Execute executes a test case and returns an error if occurred
func (h *remoteWorkflowList) Execute(workflowList string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprintf(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
_, _ = fmt.Fprintf(w, "This is test headers matcher text")
}
})
router.GET("/workflow_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(workflowList)
if err != nil {
w.WriteHeader(500)
}
_, err = w.Write(file)
if err != nil {
w.WriteHeader(500)
}
})
ts := httptest.NewServer(router)
defer ts.Close()
configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
if err != nil {
return err
}
defer func() {
_ = os.Remove("test-config.yaml")
}()
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/workflow_list", "-config", "test-config.yaml")
if err != nil {
return err
}
return expectResultsCount(results, 3)
}
type nonExistentTemplateList struct{}
// Execute executes a test case and returns an error if occurred
func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error {
router := httprouter.New()
ts := httptest.NewServer(router)
defer ts.Close()
configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
if err != nil {
return err
}
defer func() {
_ = os.Remove("test-config.yaml")
}()
_, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-template-url", ts.URL+"/404", "-config", "test-config.yaml")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
return nil
}
type nonExistentWorkflowList struct{}
// Execute executes a test case and returns an error if occurred
func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error {
router := httprouter.New()
ts := httptest.NewServer(router)
defer ts.Close()
configFileData := `remote-template-domain: [ "` + ts.Listener.Addr().String() + `" ]`
err := os.WriteFile("test-config.yaml", []byte(configFileData), permissionutil.ConfigFilePermission)
if err != nil {
return err
}
defer func() {
_ = os.Remove("test-config.yaml")
}()
_, err = testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", ts.URL, "-workflow-url", ts.URL+"/404", "-config", "test-config.yaml")
if err == nil {
return fmt.Errorf("expected error for nonexisting workflow url")
}
return nil
}
type loadTemplateWithID struct{}
func (h *loadTemplateWithID) Execute(nooop string) error {
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, nil, "-target", "scanme.sh", "-id", "self-signed-ssl")
if err != nil {
return errkit.Wrap(err, "failed to load template with id")
}
return expectResultsCount(results, 1)
}
cmd/integration-test/network.go
-241
View File
@@ -1,241 +0,0 @@
package main
import (
"fmt"
"net"
"os"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
osutils "github.com/projectdiscovery/utils/os"
"github.com/projectdiscovery/utils/reader"
)
var networkTestcases = []TestCaseInfo{
{Path: "protocols/network/basic.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }},
{Path: "protocols/network/hex.yaml", TestCase: &networkBasic{}, DisableOn: func() bool { return osutils.IsWindows() }},
{Path: "protocols/network/multi-step.yaml", TestCase: &networkMultiStep{}},
{Path: "protocols/network/self-contained.yaml", TestCase: &networkRequestSelContained{}},
{Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}},
{Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}},
{Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}},
{Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}},
{Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}},
}
const defaultStaticPort = 5431
type networkBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkBasic) Execute(filePath string) error {
var routerErr error
ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
if err != nil {
routerErr = err
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
} else {
routerErr = fmt.Errorf("invalid data received: %s", string(data))
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err)
return err
}
if routerErr != nil {
_, _ = fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr)
return routerErr
}
return expectResultsCount(results, 1)
}
type networkMultiStep struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkMultiStep) Execute(filePath string) error {
var routerErr error
ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second)
if err != nil {
routerErr = err
return
}
if string(data) == "FIRST" {
_, _ = conn.Write([]byte("PING"))
}
data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second)
if err != nil {
routerErr = err
return
}
if string(data) == "SECOND" {
_, _ = conn.Write([]byte("PONG"))
}
_, _ = conn.Write([]byte("NUCLEI"))
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
if routerErr != nil {
return routerErr
}
var expectedResultsSize int
if debug {
expectedResultsSize = 3
} else {
expectedResultsSize = 1
}
return expectResultsCount(results, expectedResultsSize)
}
type networkRequestSelContained struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkRequestSelContained) Execute(filePath string) error {
ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
_, _ = conn.Write([]byte("Authentication successful"))
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type networkVariables struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkVariables) Execute(filePath string) error {
var routerErr error
ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
if err != nil {
routerErr = err
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("aGVsbG8="))
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
if routerErr != nil {
return routerErr
}
return expectResultsCount(results, 1)
}
type networkPort struct{}
func (n *networkPort) Execute(filePath string) error {
ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
if err != nil {
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err != nil {
return err
}
// even though we passed port 443 in url it is ignored and port 23846 is used
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "23846", "443"), debug)
if err != nil {
return err
}
if err := expectResultsCount(results, 1); err != nil {
return err
}
// this is positive test case where we expect port to be overridden and 34567 to be used
ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
if err != nil {
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
defer ts2.Close()
// even though we passed port 443 in url it is ignored and port 23846 is used
// instead of hardcoded port 23846 in template
results, err = testutils.RunNucleiTemplateAndGetResults(filePath, ts2.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type networkhttps struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkhttps) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
cmd/integration-test/ssl.go
-157
View File
@@ -1,157 +0,0 @@
package main
import (
"crypto/tls"
"net"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var sslTestcases = []TestCaseInfo{
{Path: "protocols/ssl/basic.yaml", TestCase: &sslBasic{}},
{Path: "protocols/ssl/basic-ztls.yaml", TestCase: &sslBasicZtls{}},
{Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}},
{Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}},
{Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}},
{Path: "protocols/ssl/multi-req.yaml", TestCase: &sslMultiReq{}},
}
type sslBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *sslBasic) Execute(filePath string) error {
ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type sslBasicZtls struct{}
// Execute executes a test case and returns an error if occurred
func (h *sslBasicZtls) Execute(filePath string) error {
ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ztls")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type sslCustomCipher struct{}
// Execute executes a test case and returns an error if occurred
func (h *sslCustomCipher) Execute(filePath string) error {
ts := testutils.NewTCPServer(&tls.Config{CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type sslCustomVersion struct{}
// Execute executes a test case and returns an error if occurred
func (h *sslCustomVersion) Execute(filePath string) error {
ts := testutils.NewTCPServer(&tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type sslWithVars struct{}
func (h *sslWithVars) Execute(filePath string) error {
ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V", "test=asdasdas")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type sslMultiReq struct{}
func (h *sslMultiReq) Execute(filePath string) error {
//nolint:staticcheck // SSLv3 is intentionally used for testing purposes
ts := testutils.NewTCPServer(&tls.Config{
MinVersion: tls.VersionSSL30,
MaxVersion: tls.VersionTLS11,
}, defaultStaticPort, func(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
return
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V")
if err != nil {
return err
}
return expectResultsCount(results, 2)
}
cmd/integration-test/template-dir.go
-32
View File
@@ -1,32 +0,0 @@
package main
import (
"os"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/utils/errkit"
)
var templatesDirTestCases = []TestCaseInfo{
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templateDirWithTargetTest{}},
}
type templateDirWithTargetTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templateDirWithTargetTest) Execute(filePath string) error {
tempdir, err := os.MkdirTemp("", "nuclei-update-dir-*")
if err != nil {
return errkit.Wrap(err, "failed to create temp dir")
}
defer func() {
_ = os.RemoveAll(tempdir)
}()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "8x8exch02.8x8.com", debug, "-ud", tempdir)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
cmd/integration-test/templates-dir-env.go
-217
View File
@@ -1,217 +0,0 @@
package main
import (
"os"
"path/filepath"
osutils "github.com/projectdiscovery/utils/os"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/utils/errkit"
)
// isNotLinux returns true if not running on Linux (used to skip tests on non-Linux OS)
var isNotLinux = func() bool { return !osutils.IsLinux() }
var templatesDirEnvTestCases = []TestCaseInfo{
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvBasicTest{}, DisableOn: isNotLinux},
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvAbsolutePathTest{}, DisableOn: isNotLinux},
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvRelativePathTest{}, DisableOn: isNotLinux},
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvPrecedenceTest{}, DisableOn: isNotLinux},
{Path: "protocols/dns/cname-fingerprint.yaml", TestCase: &templatesDirEnvCustomTemplatesTest{}, DisableOn: isNotLinux},
}
// copyTemplateToDir copies a template file to a destination directory, preserving the directory structure
func copyTemplateToDir(templatePath, destDir string) error {
// Read the template file
templateData, err := os.ReadFile(templatePath)
if err != nil {
return errkit.Wrap(err, "failed to read template file")
}
// Create the destination path preserving directory structure
destPath := filepath.Join(destDir, templatePath)
destDirPath := filepath.Dir(destPath)
// Create the destination directory if it doesn't exist
if err := os.MkdirAll(destDirPath, 0755); err != nil {
return errkit.Wrap(err, "failed to create destination directory")
}
// Write the template file
if err := os.WriteFile(destPath, templateData, 0644); err != nil {
return errkit.Wrap(err, "failed to write template file")
}
return nil
}
// templatesDirEnvBasicTest tests basic functionality of NUCLEI_TEMPLATES_DIR
type templatesDirEnvBasicTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templatesDirEnvBasicTest) Execute(filePath string) error {
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*")
if err != nil {
return errkit.Wrap(err, "failed to create temp dir")
}
defer func() {
_ = os.RemoveAll(tempdir)
}()
// Copy template to temp directory
if err := copyTemplateToDir(filePath, tempdir); err != nil {
return err
}
// Set NUCLEI_TEMPLATES_DIR and run nuclei
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir}
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// templatesDirEnvAbsolutePathTest tests that absolute paths work correctly
type templatesDirEnvAbsolutePathTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templatesDirEnvAbsolutePathTest) Execute(filePath string) error {
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-env-abs-*")
if err != nil {
return errkit.Wrap(err, "failed to create temp dir")
}
defer func() {
_ = os.RemoveAll(tempdir)
}()
// Get absolute path
absTempDir, err := filepath.Abs(tempdir)
if err != nil {
return errkit.Wrap(err, "failed to get absolute path")
}
// Copy template to temp directory
if err := copyTemplateToDir(filePath, absTempDir); err != nil {
return err
}
// Set NUCLEI_TEMPLATES_DIR with absolute path and run nuclei
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + absTempDir}
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// templatesDirEnvRelativePathTest tests that relative paths are resolved correctly
type templatesDirEnvRelativePathTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templatesDirEnvRelativePathTest) Execute(filePath string) error {
// Create temp directory in current working directory
tempdir, err := os.MkdirTemp(".", "nuclei-templates-dir-env-rel-*")
if err != nil {
return errkit.Wrap(err, "failed to create temp dir")
}
defer func() {
_ = os.RemoveAll(tempdir)
}()
// Get relative path (just the directory name)
relPath := filepath.Base(tempdir)
// Copy template to temp directory
if err := copyTemplateToDir(filePath, tempdir); err != nil {
return err
}
// Set NUCLEI_TEMPLATES_DIR with relative path and run nuclei
// Note: The implementation should convert relative paths to absolute
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + relPath}
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// templatesDirEnvPrecedenceTest tests that -ud flag takes precedence over NUCLEI_TEMPLATES_DIR
type templatesDirEnvPrecedenceTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templatesDirEnvPrecedenceTest) Execute(filePath string) error {
// Create two temp directories
envTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-env-*")
if err != nil {
return errkit.Wrap(err, "failed to create env temp dir")
}
defer func() {
_ = os.RemoveAll(envTempDir)
}()
flagTempDir, err := os.MkdirTemp("", "nuclei-templates-dir-flag-*")
if err != nil {
return errkit.Wrap(err, "failed to create flag temp dir")
}
defer func() {
_ = os.RemoveAll(flagTempDir)
}()
// Copy template to flag temp directory (this should be used due to precedence)
if err := copyTemplateToDir(filePath, flagTempDir); err != nil {
return err
}
// Set NUCLEI_TEMPLATES_DIR to envTempDir (should be ignored due to -ud flag)
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + envTempDir}
// Use -ud flag which should take precedence
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com", "-ud", flagTempDir)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
// templatesDirEnvCustomTemplatesTest tests that custom template subdirectories are correctly set
type templatesDirEnvCustomTemplatesTest struct{}
// Execute executes a test case and returns an error if occurred
func (h *templatesDirEnvCustomTemplatesTest) Execute(filePath string) error {
tempdir, err := os.MkdirTemp("", "nuclei-templates-dir-custom-*")
if err != nil {
return errkit.Wrap(err, "failed to create temp dir")
}
defer func() {
_ = os.RemoveAll(tempdir)
}()
// Create custom template subdirectories structure
customDirs := []string{"github", "s3", "gitlab", "azure"}
for _, dir := range customDirs {
customDirPath := filepath.Join(tempdir, dir)
if err := os.MkdirAll(customDirPath, 0755); err != nil {
return errkit.Wrap(err, "failed to create custom template directory")
}
}
// Copy template to temp directory
if err := copyTemplateToDir(filePath, tempdir); err != nil {
return err
}
// Set NUCLEI_TEMPLATES_DIR and run nuclei
envVars := []string{"NUCLEI_TEMPLATES_DIR=" + tempdir}
results, err := testutils.RunNucleiBareArgsAndGetResults(debug, envVars, "-t", filePath, "-u", "8x8exch02.8x8.com")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
cmd/integration-test/websocket.go
-107
View File
@@ -1,107 +0,0 @@
package main
import (
"net"
"strings"
"github.com/gobwas/ws/wsutil"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var websocketTestCases = []TestCaseInfo{
{Path: "protocols/websocket/basic.yaml", TestCase: &websocketBasic{}},
{Path: "protocols/websocket/cswsh.yaml", TestCase: &websocketCswsh{}},
{Path: "protocols/websocket/no-cswsh.yaml", TestCase: &websocketNoCswsh{}},
{Path: "protocols/websocket/path.yaml", TestCase: &websocketWithPath{}},
}
type websocketBasic struct{}
// Execute executes a test case and returns an error if occurred
func (h *websocketBasic) Execute(filePath string) error {
connHandler := func(conn net.Conn) {
for {
msg, op, _ := wsutil.ReadClientData(conn)
if string(msg) != "hello" {
return
}
_ = wsutil.WriteServerMessage(conn, op, []byte("world"))
}
}
originValidate := func(origin string) bool {
return true
}
ts := testutils.NewWebsocketServer("", connHandler, originValidate)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type websocketCswsh struct{}
// Execute executes a test case and returns an error if occurred
func (h *websocketCswsh) Execute(filePath string) error {
connHandler := func(conn net.Conn) {
}
originValidate := func(origin string) bool {
return true
}
ts := testutils.NewWebsocketServer("", connHandler, originValidate)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type websocketNoCswsh struct{}
// Execute executes a test case and returns an error if occurred
func (h *websocketNoCswsh) Execute(filePath string) error {
connHandler := func(conn net.Conn) {
}
originValidate := func(origin string) bool {
return origin == "https://google.com"
}
ts := testutils.NewWebsocketServer("", connHandler, originValidate)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
type websocketWithPath struct{}
// Execute executes a test case and returns an error if occurred
func (h *websocketWithPath) Execute(filePath string) error {
connHandler := func(conn net.Conn) {
}
originValidate := func(origin string) bool {
return origin == "https://google.com"
}
ts := testutils.NewWebsocketServer("/test", connHandler, originValidate)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug)
if err != nil {
return err
}
return expectResultsCount(results, 0)
}
cmd/tools/fuzzplayground/main.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
_ "github.com/mattn/go-sqlite3"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils/fuzzplayground"
"github.com/projectdiscovery/nuclei/v3/internal/fuzzplayground"
)
var (
cmd/tools/signer/main.go
+19
View File
@@ -52,6 +52,17 @@ func main() {
if err != nil {
gologger.Fatal().Msgf("failed to create signer: %s", err)
}
certData, err := os.ReadFile(cert)
if err != nil {
gologger.Fatal().Msgf("failed to read certificate file %s: %s", cert, err)
}
tmplVerifier, err := signer.NewTemplateSigVerifier(certData)
if err != nil {
gologger.Fatal().Msgf("failed to create verifier: %s", err)
}
if err := signer.AddSignerToDefault(tmplVerifier); err != nil {
gologger.Fatal().Msgf("failed to register verifier: %s", err)
}
gologger.Info().Msgf("Template Signer: %v\n", tmplSigner.Identifier())
// read file
@@ -95,6 +106,14 @@ func main() {
gologger.Info().Msgf("------------------------")
gologger.Info().Msgf("Signature: %s", sig2)
gologger.Info().Msgf("Content Hash (SHA256): %s\n", hex.EncodeToString(hash2[:]))
tmpl, err = templates.Parse(template, nil, execOpts)
if err != nil {
gologger.Fatal().Msgf("failed to parse signed template: %s", err)
}
}
if !tmpl.Verified {
gologger.Fatal().Msg("template signature could not be verified with the provided certificate")
}
gologger.Info().Msgf("✓ Template signed & verified successfully")
}
gh_retry.sh
-105
View File
@@ -1,105 +0,0 @@
#!/bin/bash
# This script is used to retry failed workflows in github actions.
# It uses gh cli to fetch the failed workflows and then rerun them.
# It also checks the logs of the failed workflows to see if it is a flaky test.
# If it is a flaky test, it will rerun the failed jobs in the workflow.
# eg:
# ./gh_retry.sh -h to see the help.
# ./gh_retry.sh will run the script with default values.
# You can also pass the following arguments:
# ./gh_retry.sh -b master -l 30 -t "30 mins ago" -w "Build Test"
#Initialize variables to default values.
BRANCH=$(git symbolic-ref --short HEAD)
LIMIT=30
BEFORE="30 mins ago"
WORKFLOW="Build Test"
# You can add multiple patterns separated by |
GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed'
#Set fonts for Help.
NORM=$(tput sgr0)
BOLD=$(tput bold)
REV=$(tput smso)
HELP()
{
# Display Help
echo "Script to retry failed workflows in github actions."
echo
echo "Syntax: scriptTemplate [-b]"
echo "options:"
echo "${REV}-b${NORM} Branch to check failed workflows/jobs. Default is ${BOLD}$BRANCH${NORM}."
echo "${REV}-l${NORM} Maximum number of runs to fetch. Default is ${BOLD}$LIMIT${NORM}."
echo "${REV}-t${NORM} Time to filter the failed jobs. Default is ${BOLD}$BEFORE${NORM}."
echo "${REV}-w${NORM} Workflow to filter the failed jobs. Default is ${BOLD}$WORKFLOW${NORM}."
echo
}
while getopts :b:l:t:w:h FLAG; do
case $FLAG in
b)
BRANCH=$OPTARG
;;
l)
LIMIT=$OPTARG
;;
t)
BEFORE=$OPTARG
;;
w)
WORKFLOW=$OPTARG
;;
h) #show help
HELP
exit 0
;;
\?) #unrecognized option - show help
echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed."
HELP
exit 0
;;
esac
done
shift $((OPTIND-1))
function print_bold() {
echo "${BOLD}$1${NORM}"
}
function retry_failed_jobs() {
print_bold "Checking failed workflows for branch $BRANCH before $BEFORE"
date=$(date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE")
workflowIds=$(gh run list --limit "$LIMIT" --json headBranch,status,name,conclusion,databaseId,updatedAt | jq -c '.[] |
select ( .headBranch==$branch ) |
select ( .name | contains($workflow) ) |
select ( .conclusion=="failure" ) |
select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" --arg workflow "$WORKFLOW" | jq .databaseId)
# convert line separated by space to array
eval "arr=($workflowIds)"
if [[ -z $arr ]]
then
print_bold "Could not find any failed workflows in the last $BEFORE"
exit 0
fi
for s in "${arr[@]}"; do
print_bold "Checking logs of failed workflow $s to see if it is a flaky test"
gh run view "$s" --log-failed | grep -E "$GREP_ERROR_PATTERN" > /dev/null
if [ $? == 0 ] ; then
print_bold "Retrying failed jobs $s"
gh run rerun "$s" --failed
sleep 10s
gh run view "$s"
fi
done
}
retry_failed_jobs
integration_tests/debug.sh
-51
View File
@@ -1,51 +0,0 @@
#!/bin/bash
if [ $1 = "-h" ]; then
echo "Help for ./debug.sh"
printf "\n1. To run all integration tests of 'x' protocol:\n"
printf " \$ ./debug.sh http\n\n"
printf "2. To run all integration tests of 'x' protocol that contains 'y' in template name:\n"
printf " \$ ./debug.sh http self\n\n"
printf "3. To run all integration tests of 'x' protocol that contains 'y' in template name and pass extra args to nuclei:\n"
printf " \$ ./debug.sh http self -svd -debug-req\n\n"
printf "nuclei binary is created every time script is run but integration-test binary is not"
exit 0
fi
# Stop execution if race condition is found
export GORACE="halt_on_error=1"
echo "::group::Build nuclei"
rm nuclei 2>/dev/null
cd ../cmd/nuclei
go build -race .
mv nuclei ../../integration_tests/nuclei
echo -e "::endgroup::\n"
cd ../../integration_tests
cmdstring=""
if [ -n "$1" ]; then
cmdstring+=" -protocol $1 "
fi
if [ -n "$2" ]; then
cmdstring+=" -template $2 "
fi
# Parse any extra args that are directly passed to nuclei
if [ -n $debugArgs ]; then
export DebugExtraArgs="${@:3}"
fi
echo "$DebugExtraArgs"
echo -e "::group::Run Integration Test\n"
./integration-test $cmdstring
if [ $? -eq 0 ]
then
echo -e "::endgroup::\n"
exit 0
else
exit 1
fi
integration_tests/run.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
echo "::group::Build nuclei"
rm integration-test fuzzplayground nuclei 2>/dev/null
cd ../cmd/nuclei
go build -race .
mv nuclei ../../integration_tests/nuclei
echo "::endgroup::"
echo "::group::Build nuclei integration-test"
cd ../integration-test
go build
mv integration-test ../../integration_tests/integration-test
cd ../../integration_tests
echo "::endgroup::"
echo "::group::Installing nuclei templates"
./nuclei -update-templates
echo "::endgroup::"
./integration-test
if [ $? -eq 0 ]
then
exit 0
else
exit 1
fi
pkg/testutils/fuzzplayground/db.go → internal/fuzzplayground/db.go
View File
pkg/testutils/fuzzplayground/server.go → internal/fuzzplayground/server.go
View File
pkg/testutils/fuzzplayground/sqli_test.go → internal/fuzzplayground/sqli_test.go
View File
internal/runner/runner_test.go
+3 -2
View File
@@ -2,6 +2,7 @@ package runner
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@@ -12,14 +13,14 @@ import (
func TestCreateReportingOptions(t *testing.T) {
var options types.Options
options.ReportingConfig = "../../integration_tests/test-issue-tracker-config1.yaml"
options.ReportingConfig = filepath.Join("testdata", "test-issue-tracker-config1.yaml")
resultOptions, err := createReportingOptions(&options)
require.Nil(t, err)
require.Equal(t, resultOptions.AllowList.Severities, severity.Severities{severity.High, severity.Critical})
require.Equal(t, resultOptions.DenyList.Severities, severity.Severities{severity.Low})
options.ReportingConfig = "../../integration_tests/test-issue-tracker-config2.yaml"
options.ReportingConfig = filepath.Join("testdata", "test-issue-tracker-config2.yaml")
resultOptions2, err := createReportingOptions(&options)
require.Nil(t, err)
require.Equal(t, resultOptions2.AllowList.Severities, resultOptions.AllowList.Severities)
internal/runner/testdata/test-issue-tracker-config1.yaml
Vendored
+68
View File
@@ -0,0 +1,68 @@
allow-list:
severity: high, critical
deny-list:
severity: low
# GitHub contains configuration options for GitHub issue tracker
github:
# base-url is the optional self-hosted GitHub application url
base-url: https://localhost:8443/github
# username is the username of the GitHub user
username: test-username
# owner is the owner name of the repository for issues
owner: test-owner
# token is the token for GitHub account
token: test-token
# project-name is the name of the repository
project-name: test-project
# issue-label is the label of the created issue type
issue-label: bug
# GitLab contains configuration options for gitlab issue tracker
gitlab:
# base-url is the optional self-hosted GitLab application url
base-url: https://localhost:8443/gitlab
# username is the username of the GitLab user
username: test-username
# token is the token for GitLab account
token: test-token
# project-name is the name/id of the project(repository)
project-name: "1234"
# issue-label is the label of the created issue type
issue-label: bug
# Jira contains configuration options for Jira issue tracker
jira:
# cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
cloud: true
# update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created
update-existing: false
# URL is the jira application url
url: https://localhost/jira
# account-id is the account-id of the Jira user or username in case of on-prem Jira
account-id: test-account-id
# email is the email of the user for Jira instance
email: test@test.com
# token is the token for Jira instance or password in case of on-prem Jira
token: test-token
# project-name is the name of the project.
project-name: test-project-name
# issue-type is the name of the created issue type
issue-type: bug
# elasticsearch contains configuration options for elasticsearch exporter
elasticsearch:
# IP for elasticsearch instance
ip: 127.0.0.1
# Port is the port of elasticsearch instance
port: 9200
# IndexName is the name of the elasticsearch index
index-name: nuclei
# SSL enables ssl for elasticsearch connection
ssl: false
# SSLVerification disables SSL verification for elasticsearch
ssl-verification: false
# Username for the elasticsearch instance
username: test
# Password is the password for elasticsearch instance
password: test
integration_tests/test-issue-tracker-config2.yaml → internal/runner/testdata/test-issue-tracker-config2.yaml
Vendored
View File
internal/tests/functional/functional_test.go
+499
View File
@@ -0,0 +1,499 @@
//go:build functional
// +build functional
package functional_test
import (
"bufio"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"testing"
"github.com/google/shlex"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
type functionalHarness struct {
repoRoot string
testcasesPath string
configDir string
releaseBinary string
currentBinary string
releaseRunner *testutils.Runner
currentRunner *testutils.Runner
debug bool
ci bool
runner *testutils.Runner
}
var suite functionalHarness
func TestMain(m *testing.M) {
os.Exit(runTestMain(m))
}
func runTestMain(m *testing.M) int {
wd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to determine working directory: %v\n", err)
return 1
}
repoRoot := filepath.Clean(filepath.Join(wd, "../../.."))
configDir, err := os.MkdirTemp("", "nuclei-functional-config-*")
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create functional config dir: %v\n", err)
return 1
}
suite = functionalHarness{
repoRoot: repoRoot,
testcasesPath: filepath.Join(repoRoot, "internal", "tests", "functional", "testdata", "testcases.txt"),
configDir: configDir,
debug: isDebugMode(),
ci: isCI(),
runner: testutils.NewRunner(
testutils.WithWorkingDir(repoRoot),
testutils.WithBaseEnv("NUCLEI_CONFIG_DIR="+configDir),
),
}
defer func() {
_ = os.RemoveAll(configDir)
}()
if suite.ci {
releaseBinary, err := resolveFunctionalBinary(os.Getenv("RELEASE_BINARY"), "nuclei")
if err != nil {
fmt.Fprintf(os.Stderr, "failed to find release nuclei binary on PATH: %v\n", err)
return 1
}
currentBinary, err := resolveCurrentFunctionalBinary(repoRoot)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
suite.releaseBinary = releaseBinary
suite.currentBinary = currentBinary
suite.releaseRunner = suite.runner.Clone(testutils.WithBinaryPath(releaseBinary))
suite.currentRunner = suite.runner.Clone(testutils.WithBinaryPath(currentBinary))
if err := prepareFunctionalEnvironment(currentBinary, repoRoot, configDir); err != nil {
fmt.Fprintln(os.Stderr, err)
return 1
}
}
return m.Run()
}
func TestFunctionalComparison(t *testing.T) {
if !suite.ci {
t.Skip("functional suite requires CI=true; use make functional-dev for local release-vs-dev comparisons")
}
testcases, err := loadFunctionalTestcases(suite.testcasesPath)
if err != nil {
t.Fatalf("failed to load functional testcases: %v", err)
}
semaphore := make(chan struct{}, functionalParallelism())
for index, testcase := range testcases {
testcase := testcase
name := functionalTestName(index, testcase.raw)
t.Run(name, func(t *testing.T) {
t.Parallel()
semaphore <- struct{}{}
defer func() { <-semaphore }()
if err := runFunctionalCase(testcase.args, suite.debug); err != nil {
t.Fatal(err)
}
})
}
}
type functionalTestcase struct {
raw string
args []string
}
func loadFunctionalTestcases(path string) ([]functionalTestcase, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
var testcases []functionalTestcase
seen := make(map[string]struct{})
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if _, ok := seen[line]; ok {
continue
}
args := splitFunctionalArgs(line)
if len(args) == 0 {
continue
}
seen[line] = struct{}{}
testcases = append(testcases, functionalTestcase{raw: line, args: args})
}
if err := scanner.Err(); err != nil {
return nil, err
}
return testcases, nil
}
func splitFunctionalArgs(testcase string) []string {
parts, err := shlex.Split(testcase)
if err != nil {
return nil
}
if len(parts) <= 1 {
return nil
}
args := parts[1:]
restoreRawFlagValues(args, "-tc", extractRawFlagValues(testcase, "-tc"))
return args
}
func extractRawFlagValues(testcase, flagName string) []string {
values := make([]string, 0, 1)
for searchStart := 0; searchStart < len(testcase); {
flagIndex := findFlagIndex(testcase[searchStart:], flagName)
if flagIndex == -1 {
break
}
flagStart := searchStart + flagIndex
valueStart := flagStart + len(flagName)
for valueStart < len(testcase) && isFunctionalArgSpace(testcase[valueStart]) {
valueStart++
}
if valueStart >= len(testcase) {
break
}
valueEnd := scanFlagValueEnd(testcase, valueStart)
values = append(values, trimMatchingQuotes(strings.TrimSpace(testcase[valueStart:valueEnd])))
searchStart = valueEnd
}
return values
}
func findFlagIndex(input, flagName string) int {
for searchIndex := 0; searchIndex < len(input); {
candidate := strings.Index(input[searchIndex:], flagName)
if candidate == -1 {
return -1
}
candidate += searchIndex
if functionalFlagBoundary(input, candidate, len(flagName)) {
return candidate
}
searchIndex = candidate + len(flagName)
}
return -1
}
func functionalFlagBoundary(input string, start, length int) bool {
if start > 0 && !isFunctionalArgSpace(input[start-1]) {
return false
}
end := start + length
return end == len(input) || isFunctionalArgSpace(input[end])
}
func scanFlagValueEnd(input string, start int) int {
quote := byte(0)
for index := start; index < len(input); index++ {
char := input[index]
if quote != 0 {
if char == quote {
quote = 0
}
continue
}
switch char {
case '\'', '"':
quote = char
case ' ', '\t', '\n', '\r':
next := index
for next < len(input) && isFunctionalArgSpace(input[next]) {
next++
}
if next < len(input) && input[next] == '-' && next+1 < len(input) && isFunctionalFlagStart(input[next+1]) {
return index
}
}
}
return len(input)
}
func isFunctionalArgSpace(char byte) bool {
switch char {
case ' ', '\t', '\n', '\r':
return true
default:
return false
}
}
func isFunctionalFlagStart(char byte) bool {
return (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z')
}
func restoreRawFlagValues(args []string, flagName string, rawValues []string) {
if len(rawValues) == 0 {
return
}
rawIndex := 0
for index := 0; index < len(args)-1 && rawIndex < len(rawValues); index++ {
if args[index] != flagName {
continue
}
args[index+1] = rawValues[rawIndex]
rawIndex++
}
}
func trimMatchingQuotes(value string) string {
if len(value) < 2 {
return value
}
if (value[0] == '\'' && value[len(value)-1] == '\'') || (value[0] == '"' && value[len(value)-1] == '"') {
return value[1 : len(value)-1]
}
return value
}
func TestSplitFunctionalArgs(t *testing.T) {
t.Run("quoted values", func(t *testing.T) {
args := splitFunctionalArgs(`{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical`)
expected := []string{"-tags", "cve,exposure", "-author", "geeknik,pdteam", "-severity", "high,critical"}
if strings.Join(args, "\x00") != strings.Join(expected, "\x00") {
t.Fatalf("unexpected args: got=%q want=%q", args, expected)
}
})
t.Run("tc expression preserves quotes", func(t *testing.T) {
args := splitFunctionalArgs(`{{binary}} -tags cve -author geeknik,pdteam -tc severity=='high'`)
expected := []string{"-tags", "cve", "-author", "geeknik,pdteam", "-tc", "severity=='high'"}
if strings.Join(args, "\x00") != strings.Join(expected, "\x00") {
t.Fatalf("unexpected args: got=%q want=%q", args, expected)
}
})
t.Run("tc expression before next flag", func(t *testing.T) {
args := splitFunctionalArgs(`{{binary}} -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml`)
expected := []string{"-tc", "contains(tags,'cve')", "-exclude-templates", "http/cves/2020/CVE-2020-9757.yaml"}
if strings.Join(args, "\x00") != strings.Join(expected, "\x00") {
t.Fatalf("unexpected args: got=%q want=%q", args, expected)
}
})
}
func functionalTestName(index int, raw string) string {
name := raw
name = strings.TrimPrefix(name, "{{binary}}")
name = strings.TrimSpace(name)
name = strings.ReplaceAll(name, " ", "_")
name = strings.ReplaceAll(name, "/", "-")
if len(name) > 80 {
name = name[:80]
}
if name == "" {
name = "binary"
}
return fmt.Sprintf("%03d_%s", index+1, name)
}
func functionalParallelism() int {
for _, envName := range []string{"FUNCTIONAL_PARALLELISM", "PARALLELISM"} {
if value := strings.TrimSpace(os.Getenv(envName)); value != "" {
if parsed, err := strconv.Atoi(value); err == nil && parsed > 0 {
return parsed
}
}
}
processBudget := max(min(runtime.GOMAXPROCS(0), 4), 1)
return max((processBudget+1)/2, 1)
}
func runFunctionalCase(args []string, debug bool) error {
if suite.releaseRunner == nil || suite.currentRunner == nil {
return fmt.Errorf("functional runners are not initialized")
}
var (
releaseOutput string
currentOutput string
releaseErr error
currentErr error
)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
releaseOutput, releaseErr = suite.releaseRunner.LoadedTemplates("", debug, args)
}()
go func() {
defer wg.Done()
currentOutput, currentErr = suite.currentRunner.LoadedTemplates("", debug, args)
}()
wg.Wait()
if releaseErr != nil {
return fmt.Errorf("could not run release nuclei test: %w", releaseErr)
}
if currentErr != nil {
return fmt.Errorf("could not run current nuclei test: %w", currentErr)
}
if releaseOutput != currentOutput {
return fmt.Errorf("release loaded %s templates but current loaded %s", releaseOutput, currentOutput)
}
return nil
}
func buildCurrentBinary(repoRoot, binaryPath string) error {
if err := os.MkdirAll(filepath.Dir(binaryPath), 0755); err != nil {
return fmt.Errorf("failed to create bin directory: %w", err)
}
cmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/nuclei")
cmd.Dir = repoRoot
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to build current nuclei binary: %w\n%s", err, strings.TrimSpace(string(output)))
}
return nil
}
func resolveCurrentFunctionalBinary(repoRoot string) (string, error) {
configuredBinary := strings.TrimSpace(os.Getenv("DEV_BINARY"))
if configuredBinary != "" {
if resolved, ok := resolveExistingBinaryPath(configuredBinary); ok {
return resolved, nil
}
}
currentBinary := filepath.Join(repoRoot, "bin", binaryName())
if err := buildCurrentBinary(repoRoot, currentBinary); err != nil {
return "", err
}
return currentBinary, nil
}
func resolveFunctionalBinary(configuredPath, executableName string) (string, error) {
if trimmed := strings.TrimSpace(configuredPath); trimmed != "" {
if resolved, ok := resolveExistingBinaryPath(trimmed); ok {
return resolved, nil
}
}
for _, candidate := range candidateExecutableNames(executableName) {
resolved, err := exec.LookPath(candidate)
if err == nil {
return resolved, nil
}
}
return "", fmt.Errorf("could not locate %s on PATH", executableName)
}
func resolveExistingBinaryPath(candidate string) (string, bool) {
trimmed := strings.Trim(strings.TrimSpace(candidate), `"'`)
if trimmed == "" {
return "", false
}
for _, pathCandidate := range candidateExecutableNames(trimmed) {
absolutePath, err := filepath.Abs(pathCandidate)
if err != nil {
continue
}
info, statErr := os.Stat(absolutePath)
if statErr == nil && !info.IsDir() {
return absolutePath, true
}
}
for _, pathCandidate := range candidateExecutableNames(trimmed) {
resolved, err := exec.LookPath(pathCandidate)
if err == nil {
return resolved, true
}
}
return "", false
}
func candidateExecutableNames(name string) []string {
trimmed := strings.Trim(strings.TrimSpace(name), `"'`)
if trimmed == "" {
return nil
}
candidates := []string{trimmed}
if runtime.GOOS == "windows" && filepath.Ext(trimmed) == "" {
candidates = append(candidates, trimmed+".exe")
}
return candidates
}
func prepareFunctionalEnvironment(binaryPath, repoRoot, configDir string) error {
configRoot := filepath.Join(configDir, "nuclei")
if err := os.MkdirAll(configRoot, 0755); err != nil {
return fmt.Errorf("failed to create functional nuclei config root: %w", err)
}
ignoreFile := filepath.Join(configRoot, ".nuclei-ignore")
if err := os.WriteFile(ignoreFile, nil, 0644); err != nil {
return fmt.Errorf("failed to create functional ignore file: %w", err)
}
for _, args := range [][]string{{"-update-templates"}, {"-validate"}} {
cmd := exec.Command(binaryPath, args...)
cmd.Dir = repoRoot
cmd.Env = append(os.Environ(), "NUCLEI_CONFIG_DIR="+configDir)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("functional setup command %v failed: %w\n%s", args, err, strings.TrimSpace(string(output)))
}
}
return nil
}
func binaryName() string {
if runtime.GOOS == "windows" {
return "nuclei.exe"
}
return "nuclei"
}
func isCI() bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv("CI")))
return value == "true" || value == "1"
}
func isDebugMode() bool {
for _, envVar := range []string{"DEBUG", "ACTIONS_STEP_DEBUG", "ACTIONS_RUNNER_DEBUG", "RUNNER_DEBUG"} {
if envTruthy(envVar) {
return true
}
}
return false
}
func envTruthy(name string) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(name)))
switch value {
case "true", "1", "yes", "on", "enabled":
return true
default:
return false
}
}
cmd/functional-test/targets-1000.txt → internal/tests/functional/testdata/targets-1000.txt
Vendored
View File
cmd/functional-test/targets-150.txt → internal/tests/functional/testdata/targets-150.txt
Vendored
View File
cmd/functional-test/targets-250.txt → internal/tests/functional/testdata/targets-250.txt
Vendored
View File
cmd/functional-test/targets.txt → internal/tests/functional/testdata/targets.txt
Vendored
View File
cmd/functional-test/testcases.txt → internal/tests/functional/testdata/testcases.txt
Vendored
+36 -49
View File
@@ -2,73 +2,60 @@
{{binary}}
# Template tags filter
{{binary}} -tags cve -ntv 8.8.8,8.8.9
{{binary}} -tags cve
{{binary}} -tags cve,exposure
{{binary}} -tags cve,exposure -tags token
{{binary}} -tags cve,exposure -tags token,logs
{{binary}} -tags "cve","exposure" -tags "token","logs"
{{binary}} -t http/cves/
{{binary}} -t http/cves/ -t http/exposures/
{{binary}} -t http/cves/ -t http/exposures/ -tags config
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml
{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
{{binary}} -tags cve -severity high
{{binary}} -tags cve,exposure -severity high,critical
{{binary}} -tags cve,exposure -severity high,critical,medium
{{binary}} -tags cve -author geeknik
{{binary}} -tags cve -author geeknik,pdteam
{{binary}} -tags cve -author geeknik -severity high
{{binary}} -tags cve
{{binary}} -tags cve,exposure
{{binary}} -tags cve,exposure -tags token
{{binary}} -tags cve,exposure -tags token,logs
{{binary}} -tags "cve","exposure" -tags "token","logs"
{{binary}} -tags 'cve','exposure' -tags 'token','logs'
{{binary}} -tags cve -severity high
{{binary}} -tags cve,exposure -severity high,critical
{{binary}} -tags cve,exposure -severity high,critical,medium
{{binary}} -tags cve -author geeknik
{{binary}} -tags cve -author geeknik,pdteam
{{binary}} -tags cve -author geeknik -severity high
{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical
{{binary}} -tags "cve","exposure" -tags "token","logs"
{{binary}} -tags cve
{{binary}} -tags cve -author geeknik
{{binary}} -tags cve -author geeknik -severity high
{{binary}} -tags cve -author geeknik,pdteam
{{binary}} -tags cve -etags ssrf
{{binary}} -tags cve,exposure -etags ssrf,config
{{binary}} -tags cve,exposure -etags ssrf,config -severity high
{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
{{binary}} -tags cve,dos,fuzz
{{binary}} -tags cve -include-tags dos,fuzz
{{binary}} -tags cve -exclude-tags cve2020
{{binary}} -tags cve -exclude-templates http/cves/2020/
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml
{{binary}} -tags cve -exclude-templates http/cves/2020/CVE-2020-9757.yaml -exclude-templates http/cves/2021/
{{binary}} -t http/cves/
{{binary}} -t http/cves/ -t http/exposures/
{{binary}} -t http/cves/ -t http/exposures/ -tags config
{{binary}} -t http/cves/ -t http/exposures/ -tags config,ssrf
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2021/
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates http/cves/2017/CVE-2017-7269.yaml
{{binary}} -t http/cves/ -t http/exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates http/cves/2017/CVE-2017-7269.yaml
{{binary}} -tags cve -include-tags dos,fuzz
{{binary}} -tags cve -ntv 8.8.8,8.8.9
{{binary}} -tags cve -severity high
{{binary}} -tags cve,dos,fuzz
{{binary}} -tags cve,exposure
{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
{{binary}} -tags cve,exposure -etags ssrf,config
{{binary}} -tags cve,exposure -etags ssrf,config -severity high
{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik
{{binary}} -tags cve,exposure -severity high,critical
{{binary}} -tags cve,exposure -severity high,critical,medium
{{binary}} -tags cve,exposure -tags token
{{binary}} -tags cve,exposure -tags token,logs
# Advanced Filtering
{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml
{{binary}} -tags cve -author geeknik,pdteam -tc severity=='high'
{{binary}} -tc contains(authors,'pdteam')
{{binary}} -t http/cves/ -t http/exposures/ -tc contains(tags,'cve') -exclude-templates http/cves/2020/CVE-2020-9757.yaml
{{binary}} -tc protocol=='dns'
{{binary}} -tc contains(http_method,'GET')
{{binary}} -tc len(body)>0
{{binary}} -tc contains(matcher_type,'word')
{{binary}} -tc contains(extractor_type,'regex')
{{binary}} -tc contains(description,'wordpress')
{{binary}} -tc contains(extractor_type,'regex')
{{binary}} -tc contains(http_method,'GET')
{{binary}} -tc contains(matcher_type,'word')
{{binary}} -tc len(body)>0
{{binary}} -tc protocol=='dns'
# Workflow Filters
{{binary}} -w workflows
{{binary}} -w workflows -author geeknik,pdteam
{{binary}} -w workflows -severity high,critical
{{binary}} -w workflows -author geeknik,pdteam -severity high,critical
{{binary}} -w workflows -severity high,critical
# Input Types
# http protocol
# host
{{binary}} -id tech-detect -u scanme.sh
# host:port
@@ -91,5 +78,5 @@
{{binary}} -id tls-version -u scanme.sh:22
# Options
# Tls Impersonate
# TLS Impersonate
{{binary}} -id tech-detect -tlsi -u https://scanme.sh
cmd/integration-test/code.go → internal/tests/integration/code_test.go
+70 -43
View File
@@ -1,21 +1,35 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"errors"
"log"
"os"
"path/filepath"
"os/exec"
"sync"
osutils "github.com/projectdiscovery/utils/os"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" }
var signedCodeTemplates sync.Once
var signedCodeTemplatesErr error
var codeTestCases = []TestCaseInfo{
func hasAnyExecutable(names ...string) bool {
for _, name := range names {
if _, err := exec.LookPath(name); err == nil {
return true
}
}
return false
}
var codeTestCases = []integrationCase{
{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled},
@@ -24,9 +38,9 @@ var codeTestCases = []TestCaseInfo{
{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
{Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/sh-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }},
{Path: "protocols/code/py-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() }},
{Path: "protocols/code/pwsh-echo.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return isCodeDisabled() }},
{Path: "protocols/code/sh-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() || !hasAnyExecutable("docker", "podman") }, Serial: true},
{Path: "protocols/code/py-virtual.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsLinux() || isCodeDisabled() || !hasAnyExecutable("docker", "podman") }, Serial: true},
{Path: "protocols/code/pwsh-echo.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return isCodeDisabled() || !hasAnyExecutable("pwsh", "powershell", "powershell.exe") }},
}
const (
@@ -36,49 +50,62 @@ const (
var testcertpath = ""
func init() {
func ensureSignedCodeTemplates() error {
if isCodeDisabled() {
// skip executing code protocol in CI on windows
return
return nil
}
// allow local file access to load content of file references in template
// in order to sign them for testing purposes
templates.TemplateSignerLFA()
tsigner, err := signer.NewTemplateSignerFromFiles(testCertFile, testKeyFile)
if err != nil {
panic(err)
}
testcertpath, _ = filepath.Abs(testCertFile)
for _, v := range codeTestCases {
templatePath := v.Path
testCase := v.TestCase
if v.DisableOn != nil && v.DisableOn() {
// skip ps1 test case on non-windows platforms
continue
}
templatePath, err := filepath.Abs(templatePath)
signedCodeTemplates.Do(func() {
previousWD, err := os.Getwd()
if err != nil {
panic(err)
signedCodeTemplatesErr = err
return
}
if err := os.Chdir(suite.fixturesDir); err != nil {
signedCodeTemplatesErr = err
return
}
defer func() {
if chdirErr := os.Chdir(previousWD); chdirErr != nil && signedCodeTemplatesErr == nil {
signedCodeTemplatesErr = chdirErr
}
}()
templates.TemplateSignerLFA()
certPath := fixturePath(testCertFile)
keyPath := fixturePath(testKeyFile)
testcertpath = certPath
tsigner, err := signer.NewTemplateSignerFromFiles(certPath, keyPath)
if err != nil {
signedCodeTemplatesErr = err
return
}
// skip
// - unsigned test cases
if _, ok := testCase.(*unsignedCode); ok {
continue
templatesToSign := []string{
"workflow/code-template-1.yaml",
"workflow/code-template-2.yaml",
}
if _, ok := testCase.(*codePyNoSig); ok {
continue
for _, v := range codeTestCases {
if v.DisableOn != nil && v.DisableOn() {
continue
}
if _, ok := v.TestCase.(*unsignedCode); ok {
continue
}
if _, ok := v.TestCase.(*codePyNoSig); ok {
continue
}
templatesToSign = append(templatesToSign, v.Path)
}
if err := templates.SignTemplate(tsigner, templatePath); err != nil {
log.Fatalf("Could not sign template %v got: %s\n", templatePath, err)
for _, templatePath := range templatesToSign {
if err := templates.SignTemplate(tsigner, fixturePath(templatePath)); err != nil {
signedCodeTemplatesErr = err
return
}
}
}
})
return signedCodeTemplatesErr
}
func getEnvValues() []string {
internal/tests/integration/config_and_templates_test.go
+142
View File
@@ -0,0 +1,142 @@
//go:build integration
// +build integration
package integration_test
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"slices"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func newTemplateDirTarget(t *testing.T) string {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("This is test matcher text"))
}))
t.Cleanup(server.Close)
return server.URL
}
func TestTemplateDir(t *testing.T) {
t.Run("WithTarget", func(t *testing.T) {
tempDir := t.TempDir()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/http/get.yaml", newTemplateDirTarget(t), suite.debug, "-ud", tempDir)
if err != nil {
t.Fatalf("template dir override request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
func TestTemplatesDirEnv(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("NUCLEI_TEMPLATES_DIR integration cases are only supported on Linux")
}
target := newTemplateDirTarget(t)
t.Run("Basic", func(t *testing.T) {
tempDir := t.TempDir()
copyFixtureToDir(t, "protocols/http/get.yaml", tempDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target)
if err != nil {
t.Fatalf("templates dir env basic request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("AbsolutePath", func(t *testing.T) {
tempDir := t.TempDir()
copyFixtureToDir(t, "protocols/http/get.yaml", tempDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target)
if err != nil {
t.Fatalf("templates dir env absolute path request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("RelativePath", func(t *testing.T) {
tempDir, err := os.MkdirTemp(suite.fixturesDir, "nuclei-templates-dir-env-rel-*")
if err != nil {
t.Fatalf("failed to create relative templates dir: %v", err)
}
defer func() { _ = os.RemoveAll(tempDir) }()
copyFixtureToDir(t, "protocols/http/get.yaml", tempDir)
relPath := filepath.Base(tempDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + relPath}, "-t", "protocols/http/get.yaml", "-u", target)
if err != nil {
t.Fatalf("templates dir env relative path request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("Precedence", func(t *testing.T) {
envTempDir := t.TempDir()
flagTempDir := t.TempDir()
copyFixtureToDir(t, "protocols/http/get.yaml", flagTempDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + envTempDir}, "-t", "protocols/http/get.yaml", "-u", target, "-ud", flagTempDir)
if err != nil {
t.Fatalf("templates dir env precedence request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("CustomSubdirs", func(t *testing.T) {
tempDir := t.TempDir()
for _, dir := range []string{"github", "s3", "gitlab", "azure"} {
if err := os.MkdirAll(filepath.Join(tempDir, dir), 0755); err != nil {
t.Fatalf("failed to create custom templates subdir %s: %v", dir, err)
}
}
copyFixtureToDir(t, "protocols/http/get.yaml", tempDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + tempDir}, "-t", "protocols/http/get.yaml", "-u", target)
if err != nil {
t.Fatalf("templates dir env custom subdirs request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
func TestCustomConfigDir(t *testing.T) {
t.Run("ConfigDirIsolated", func(t *testing.T) {
customTempDir := t.TempDir()
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_CONFIG_DIR=" + customTempDir}, "-t", "protocols/http/get.yaml", "-u", newTemplateDirTarget(t))
if err != nil {
t.Fatalf("custom config dir request failed: %v", err)
}
if len(results) != 0 {
files, err := os.ReadDir(customTempDir)
if err != nil {
t.Fatalf("failed to inspect custom config dir: %v", err)
}
var fileNames []string
for _, file := range files {
fileNames = append(fileNames, file.Name())
}
for _, requiredFile := range []string{".templates-config.json", "config.yaml", "reporting-config.yaml"} {
if !slices.Contains(fileNames, requiredFile) {
t.Fatalf("missing required config file %q in custom config dir: %v", requiredFile, fileNames)
}
}
}
})
}
internal/tests/integration/dns_test.go
+140
View File
@@ -0,0 +1,140 @@
//go:build integration
// +build integration
package integration_test
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func TestDNS(t *testing.T) {
t.Run("A", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/a.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns A request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("AAAA", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/aaaa.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns AAAA request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("CNAME", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/cname.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns CNAME request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("SRV", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/srv.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns SRV request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("NS", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/ns.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns NS request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("TXT", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/txt.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns TXT request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("PTR", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/ptr.yaml", "1.1.1.1", suite.debug)
if err != nil {
t.Fatalf("dns PTR request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("CAA", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/caa.yaml", "pki.goog", suite.debug)
if err != nil {
t.Fatalf("dns CAA request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("TLSA", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/tlsa.yaml", "scanme.sh", suite.debug)
if err != nil {
t.Fatalf("dns TLSA request failed: %v", err)
}
if err := expectResultsCount(results, 0); err != nil {
t.Fatal(err)
}
})
t.Run("Variables", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/variables.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns variables request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("Payload", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/payload.yaml", "google.com", suite.debug)
if err != nil {
t.Fatalf("dns payload request failed: %v", err)
}
if err := expectResultsCount(results, 3); err != nil {
t.Fatal(err)
}
results, err = testutils.RunNucleiTemplateAndGetResults("protocols/dns/payload.yaml", "google.com", suite.debug, "-var", "subdomain_wordlist=subdomains.txt")
if err != nil {
t.Fatalf("dns payload request with CLI override failed: %v", err)
}
if err := expectResultsCount(results, 4); err != nil {
t.Fatal(err)
}
})
t.Run("DSLMatcherVariable", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/dns/dsl-matcher-variable.yaml", "one.one.one.one", suite.debug)
if err != nil {
t.Fatalf("dns DSL matcher variable request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
cmd/integration-test/dsl.go → internal/tests/integration/dsl_test.go
+6 -3
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
@@ -6,10 +9,10 @@ import (
"net/http/httptest"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
var dslTestcases = []TestCaseInfo{
var dslTestcases = []integrationCase{
{Path: "dsl/hide-version-warning.yaml", TestCase: &dslVersionWarning{}},
{Path: "dsl/show-version-warning.yaml", TestCase: &dslShowVersionWarning{}},
}
cmd/integration-test/exporters.go → internal/tests/integration/exporters_test.go
+5 -2
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"context"
@@ -21,7 +24,7 @@ const (
dbImage = "mongo:8"
)
var exportersTestCases = []TestCaseInfo{
var exportersTestCases = []integrationCase{
{Path: "exporters/mongo", TestCase: &mongoExporter{}, DisableOn: func() bool {
return osutil.IsWindows() || osutil.IsOSX()
}},
internal/tests/integration/file_test.go
+55
View File
@@ -0,0 +1,55 @@
//go:build integration
// +build integration
package integration_test
import (
"path/filepath"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func TestFile(t *testing.T) {
target := filepath.ToSlash(filepath.Join("protocols", "file", "data")) + "/"
t.Run("MatcherWithOr", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-or.yaml", target, suite.debug, "-file")
if err != nil {
t.Fatalf("file OR matcher request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("MatcherWithAnd", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-and.yaml", target, suite.debug, "-file")
if err != nil {
t.Fatalf("file AND matcher request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("MatcherWithNestedAnd", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/matcher-with-nested-and.yaml", target, suite.debug, "-file")
if err != nil {
t.Fatalf("file nested AND matcher request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("Extract", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/file/extract.yaml", target, suite.debug, "-file")
if err != nil {
t.Fatalf("file extractor request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
internal/tests/integration/flow_test.go
+98
View File
@@ -0,0 +1,98 @@
//go:build integration
// +build integration
package integration_test
import (
"encoding/base64"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func TestFlow(t *testing.T) {
t.Run("ConditionalFlow", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("flow/conditional-flow.yaml", "cloud.projectdiscovery.io", suite.debug)
if err != nil {
t.Fatalf("conditional flow request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("ConditionalFlowNegative", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("flow/conditional-flow-negative.yaml", "scanme.sh", suite.debug)
if err != nil {
t.Fatalf("conditional negative flow request failed: %v", err)
}
if err := expectResultsCount(results, 0); err != nil {
t.Fatal(err)
}
})
t.Run("IterateValuesFlow", func(t *testing.T) {
router := httprouter.New()
testEmails := []string{"secrets@scanme.sh", "superadmin@scanme.sh"}
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = fmt.Fprint(w, testEmails)
})
router.GET("/user/"+getBase64(testEmails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
router.GET("/user/"+getBase64(testEmails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("Welcome ! This is test matcher text"))
})
server := httptest.NewServer(router)
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("flow/iterate-values-flow.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("iterate values flow request failed: %v", err)
}
if err := expectResultsCount(results, 2); err != nil {
t.Fatal(err)
}
})
t.Run("IterateOneValueFlow", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("flow/iterate-one-value-flow.yaml", "https://scanme.sh", suite.debug)
if err != nil {
t.Fatalf("iterate one value flow request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("DNSNSProbe", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("flow/dns-ns-probe.yaml", "oast.fun", suite.debug)
if err != nil {
t.Fatalf("dns ns probe flow request failed: %v", err)
}
if err := expectResultsCount(results, 2); err != nil {
t.Fatal(err)
}
})
t.Run("HideMatcher", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("flow/flow-hide-matcher.yaml", "scanme.sh", suite.debug)
if err != nil {
t.Fatalf("hide matcher flow request failed: %v", err)
}
if err := expectResultsCount(results, 0); err != nil {
t.Fatal(err)
}
})
}
func getBase64(input string) string {
return base64.StdEncoding.EncodeToString([]byte(input))
}
cmd/integration-test/fuzz.go → internal/tests/integration/fuzz_test.go
+6 -3
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
@@ -7,8 +10,8 @@ import (
"net/url"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
@@ -16,7 +19,7 @@ const (
targetFile = "fuzz/testData/ginandjuice.proxify.yaml"
)
var fuzzingTestCases = []TestCaseInfo{
var fuzzingTestCases = []integrationCase{
{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
{Path: "fuzz/fuzz-multi-mode.yaml", TestCase: &fuzzMultipleMode{}},
{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
cmd/integration-test/generic.go → internal/tests/integration/generic_test.go
+6 -3
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"crypto/tls"
@@ -10,11 +13,11 @@ import (
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
permissionutil "github.com/projectdiscovery/utils/permission"
)
var genericTestcases = []TestCaseInfo{
var genericTestcases = []integrationCase{
{Path: "generic/auth/certificate/http-get.yaml", TestCase: &clientCertificate{}},
}
cmd/integration-test/headless.go → internal/tests/integration/headless_test.go
+6 -3
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
@@ -8,10 +11,10 @@ import (
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
var headlessTestcases = []TestCaseInfo{
var headlessTestcases = []integrationCase{
{Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessBasic{}},
{Path: "protocols/headless/headless-waitevent.yaml", TestCase: &headlessBasic{}},
{Path: "protocols/headless/headless-dsl.yaml", TestCase: &headlessBasic{}},
cmd/integration-test/http.go → internal/tests/integration/http_test.go
+59 -38
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"errors"
@@ -7,6 +10,7 @@ import (
"net/http/httptest"
"net/http/httputil"
"os"
"path/filepath"
"reflect"
"regexp"
"strconv"
@@ -17,7 +21,7 @@ import (
"github.com/julienschmidt/httprouter"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
@@ -27,7 +31,7 @@ import (
unitutils "github.com/projectdiscovery/utils/unit"
)
var httpTestcases = []TestCaseInfo{
var httpTestcases = []integrationCase{
// TODO: excluded due to parsing errors with console
// "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{},
{Path: "protocols/http/get-headers.yaml", TestCase: &httpGetHeaders{}},
@@ -797,7 +801,7 @@ func (h *httpRawUnsafePath) Execute(filepath string) error {
// testing unsafe paths using router feedback is not possible cause they are `unsafe urls`
// hence it is done by parsing and matching paths from nuclei output with `-debug-req` flag
// read template files
bin, err := os.ReadFile(filepath)
bin, err := os.ReadFile(fixturePath(filepath))
if err != nil {
return err
}
@@ -845,7 +849,7 @@ type httpPaths struct{}
func (h *httpPaths) Execute(filepath string) error {
// covers testcases similar to httpRawUnsafePath but when `unsafe:false`
bin, err := os.ReadFile(filepath)
bin, err := os.ReadFile(fixturePath(filepath))
if err != nil {
return err
}
@@ -995,18 +999,17 @@ func (h *httpRequestSelfContained) Execute(filePath string) error {
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = w.Write([]byte("This is self-contained response"))
})
server := &http.Server{
Addr: fmt.Sprintf("localhost:%d", defaultStaticPort),
Handler: router,
}
go func() {
_ = server.ListenAndServe()
}()
defer func() {
_ = server.Close()
}()
server := httptest.NewServer(router)
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
templatePath, err := tempFixturePath(filePath, map[string]string{
"http://127.0.0.1:5431": server.URL,
"127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"),
})
if err != nil {
return err
}
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-esc")
if err != nil {
return err
}
@@ -1033,18 +1036,17 @@ func (h *httpRequestSelfContainedWithParams) Execute(filePath string) error {
}
_, _ = w.Write([]byte("This is self-contained response"))
})
server := &http.Server{
Addr: fmt.Sprintf("localhost:%d", defaultStaticPort),
Handler: router,
}
go func() {
_ = server.ListenAndServe()
}()
defer func() {
_ = server.Close()
}()
server := httptest.NewServer(router)
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-esc")
templatePath, err := tempFixturePath(filePath, map[string]string{
"http://127.0.0.1:5431": server.URL,
"127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"),
})
if err != nil {
return err
}
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-esc")
if err != nil {
return err
}
@@ -1068,16 +1070,8 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
gotReqToEndpoints = append(gotReqToEndpoints, "/two")
_, _ = w.Write([]byte("This is self-contained response"))
})
server := &http.Server{
Addr: fmt.Sprintf("localhost:%d", defaultStaticPort),
Handler: router,
}
go func() {
_ = server.ListenAndServe()
}()
defer func() {
_ = server.Close()
}()
server := httptest.NewServer(router)
defer server.Close()
// create temp file
FileLoc, err := os.CreateTemp("", "self-contained-payload-*.txt")
@@ -1091,7 +1085,14 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
_ = FileLoc.Close()
}()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc.Name(), "-esc")
templatePath, err := tempFixturePath(filePath, map[string]string{
"http://127.0.0.1:5431": server.URL,
"127.0.0.1:5431": strings.TrimPrefix(server.URL, "http://"),
})
if err != nil {
return err
}
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", debug, "-V", "test="+FileLoc.Name(), "-esc")
if err != nil {
return err
}
@@ -1106,6 +1107,26 @@ func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
return nil
}
func tempFixturePath(relativePath string, replacements map[string]string) (string, error) {
data, err := os.ReadFile(fixturePath(relativePath))
if err != nil {
return "", fmt.Errorf("failed to read fixture %s: %w", relativePath, err)
}
content := string(data)
for oldValue, newValue := range replacements {
content = strings.ReplaceAll(content, oldValue, newValue)
}
tempDir, err := os.MkdirTemp(suite.tempDir, "http-self-contained-*")
if err != nil {
return "", fmt.Errorf("failed to create temp fixture dir: %w", err)
}
tempPath := filepath.Join(tempDir, filepath.Base(relativePath))
if err := os.WriteFile(tempPath, []byte(content), 0600); err != nil {
return "", fmt.Errorf("failed to write temp fixture %s: %w", relativePath, err)
}
return tempPath, nil
}
type httpGetCaseInsensitive struct{}
// Execute executes a test case and returns an error if occurred
internal/tests/integration/integration_test.go
+279
View File
@@ -0,0 +1,279 @@
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/fuzzplayground"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
type integrationHarness struct {
repoRoot string
fixturesDir string
binaryPath string
debug bool
tempDir string
runner *testutils.Runner
}
var suite integrationHarness
var debug bool
func TestMain(m *testing.M) {
wd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to determine working directory: %v\n", err)
os.Exit(1)
}
repoRoot := filepath.Clean(filepath.Join(wd, "../../.."))
tempDir, err := os.MkdirTemp("", "nuclei-integration-*")
if err != nil {
fmt.Fprintf(os.Stderr, "failed to create integration temp dir: %v\n", err)
os.Exit(1)
}
binaryName := "nuclei"
if runtime.GOOS == "windows" {
binaryName += ".exe"
}
binaryPath := filepath.Join(tempDir, binaryName)
if err := buildNucleiBinary(repoRoot, binaryPath); err != nil {
fmt.Fprintln(os.Stderr, err)
_ = os.RemoveAll(tempDir)
os.Exit(1)
}
workingFixturesDir := filepath.Join(tempDir, "fixtures")
if err := copyDir(filepath.Join(repoRoot, "internal", "tests", "integration", "testdata"), workingFixturesDir); err != nil {
fmt.Fprintf(os.Stderr, "failed to stage integration fixtures: %v\n", err)
_ = os.RemoveAll(tempDir)
os.Exit(1)
}
previousRunner := testutils.DefaultRunner()
runner := testutils.NewRunner(
testutils.WithBinaryPath(binaryPath),
testutils.WithWorkingDir(workingFixturesDir),
testutils.WithExtraArgs(integrationExtraArgs...),
)
testutils.SetDefaultRunner(runner)
suite = integrationHarness{
repoRoot: repoRoot,
fixturesDir: workingFixturesDir,
binaryPath: binaryPath,
debug: isDebugMode(),
tempDir: tempDir,
runner: runner,
}
debug = suite.debug
fuzzServer := fuzzplayground.GetPlaygroundServer()
defer func() {
fuzzplayground.Cleanup()
_ = fuzzServer.Close()
}()
go func() {
if err := fuzzServer.Start("localhost:8082"); err != nil && !strings.Contains(err.Error(), "Server closed") {
fmt.Fprintf(os.Stderr, "failed to start fuzz playground: %v\n", err)
}
}()
exitCode := m.Run()
testutils.SetDefaultRunner(previousRunner)
_ = os.RemoveAll(tempDir)
os.Exit(exitCode)
}
func buildNucleiBinary(repoRoot, binaryPath string) error {
cmd := exec.Command("go", "build", "-o", binaryPath, "./cmd/nuclei")
cmd.Dir = repoRoot
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to build nuclei binary: %w\n%s", err, strings.TrimSpace(string(output)))
}
return nil
}
func isDebugMode() bool {
for _, envVar := range []string{"DEBUG", "ACTIONS_STEP_DEBUG", "ACTIONS_RUNNER_DEBUG", "RUNNER_DEBUG"} {
if envTruthy(envVar) {
return true
}
}
return false
}
func isCI() bool {
return envTruthy("CI")
}
func envTruthy(name string) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(name)))
switch value {
case "true", "1", "yes":
return true
default:
return false
}
}
func fixturePath(relativePath string) string {
if filepath.IsAbs(relativePath) {
return relativePath
}
return filepath.Join(suite.fixturesDir, filepath.FromSlash(relativePath))
}
func expectResultsCount(results []string, expectedNumbers ...int) error {
results = filterLines(results)
actualCount := len(results)
for _, expected := range expectedNumbers {
if actualCount == expected {
return nil
}
}
return fmt.Errorf("incorrect number of results: %d (actual) vs %v (expected)\nresults:\n\t%s", actualCount, expectedNumbers, strings.Join(results, "\n\t"))
}
type capturedError struct {
mu sync.Mutex
err error
}
func (c *capturedError) Set(err error) {
if err == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if c.err == nil {
c.err = err
}
}
func (c *capturedError) Err() error {
c.mu.Lock()
defer c.mu.Unlock()
return c.err
}
func tempFixtureCopy(t *testing.T, relativePath string, replacements map[string]string) string {
t.Helper()
data, err := os.ReadFile(fixturePath(relativePath))
if err != nil {
t.Fatalf("failed to read fixture %s: %v", relativePath, err)
}
content := string(data)
if len(replacements) > 0 {
replacementPairs := make([]string, 0, len(replacements)*2)
for oldValue, newValue := range replacements {
replacementPairs = append(replacementPairs, oldValue, newValue)
}
content = strings.NewReplacer(replacementPairs...).Replace(content)
}
tempPath := filepath.Join(t.TempDir(), filepath.Base(relativePath))
if err := os.WriteFile(tempPath, []byte(content), 0600); err != nil {
t.Fatalf("failed to write temp fixture %s: %v", relativePath, err)
}
return tempPath
}
func copyFixtureToDir(t *testing.T, relativePath, destDir string) string {
t.Helper()
srcPath := fixturePath(relativePath)
destPath := filepath.Join(destDir, filepath.FromSlash(relativePath))
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
t.Fatalf("failed to create fixture destination for %s: %v", relativePath, err)
}
if err := copyFile(srcPath, destPath, 0644); err != nil {
t.Fatalf("failed to copy fixture %s: %v", relativePath, err)
}
return destPath
}
func copyDir(srcDir, destDir string) error {
return filepath.WalkDir(srcDir, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
relPath, err := filepath.Rel(srcDir, path)
if err != nil {
return err
}
if relPath == "." {
return os.MkdirAll(destDir, 0755)
}
targetPath := filepath.Join(destDir, relPath)
if entry.IsDir() {
return os.MkdirAll(targetPath, 0755)
}
info, err := entry.Info()
if err != nil {
return err
}
return copyFile(path, targetPath, info.Mode())
})
}
func copyFile(srcPath, destPath string, mode fs.FileMode) error {
srcFile, err := os.Open(srcPath)
if err != nil {
return err
}
defer func() { _ = srcFile.Close() }()
destFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
if err != nil {
return err
}
defer func() { _ = destFile.Close() }()
_, err = io.Copy(destFile, srcFile)
return err
}
func filterLines(results []string) []string {
results = filterHeadlessLogs(results)
results = filterUnsignedTemplatesWarnings(results)
return results
}
func filterHeadlessLogs(results []string) []string {
filtered := make([]string, 0, len(results))
for _, result := range results {
if strings.Contains(result, "[launcher.Browser]") {
continue
}
filtered = append(filtered, result)
}
return filtered
}
func filterUnsignedTemplatesWarnings(results []string) []string {
filtered := make([]string, 0, len(results))
unsignedTemplatesRegex := regexp.MustCompile(`Loading \d+ unsigned templates for scan\. Use with caution\.`)
for _, result := range results {
if unsignedTemplatesRegex.MatchString(result) {
continue
}
filtered = append(filtered, result)
}
return filtered
}
cmd/integration-test/interactsh.go → internal/tests/integration/interactsh_test.go
+5 -2
View File
@@ -1,9 +1,12 @@
package main
//go:build integration
// +build integration
package integration_test
import osutils "github.com/projectdiscovery/utils/os"
// All Interactsh related testcases
var interactshTestCases = []TestCaseInfo{
var interactshTestCases = []integrationCase{
{Path: "protocols/http/interactsh.yaml", TestCase: &httpInteractshRequest{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
{Path: "protocols/http/interactsh-with-payloads.yaml", TestCase: &httpInteractshWithPayloadsRequest{}, DisableOn: func() bool { return true }},
{Path: "protocols/http/interactsh-stop-at-first-match.yaml", TestCase: &httpInteractshStopAtFirstMatchRequest{}, DisableOn: func() bool { return true }}, // disable this test for now
internal/tests/integration/javascript_test.go
+372
View File
@@ -0,0 +1,372 @@
//go:build integration
// +build integration
package integration_test
import (
"database/sql"
"fmt"
"net"
"time"
"github.com/go-pg/pg/v10"
_ "github.com/go-sql-driver/mysql"
"github.com/ory/dockertest/v3"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
osutils "github.com/projectdiscovery/utils/os"
"github.com/projectdiscovery/utils/reader"
"go.uber.org/multierr"
)
func javascriptDockerDisabled() bool {
return !osutils.IsLinux() || !hasAnyExecutable("docker", "podman")
}
var jsTestcases = []integrationCase{
{Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
{Path: "protocols/javascript/rsync-test.yaml", TestCase: &javascriptRsyncTest{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/vnc-pass-brute.yaml", TestCase: &javascriptVncPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/postgres-pass-brute.yaml", TestCase: &javascriptPostgresPassBrute{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/mysql-connect.yaml", TestCase: &javascriptMySQLConnect{}, DisableOn: javascriptDockerDisabled, Serial: true},
{Path: "protocols/javascript/multi-ports.yaml", TestCase: &javascriptMultiPortsSSH{}},
{Path: "protocols/javascript/no-port-args.yaml", TestCase: &javascriptNoPortArgs{}},
{Path: "protocols/javascript/telnet-auth-test.yaml", TestCase: &javascriptTelnetAuthTest{}, DisableOn: javascriptDockerDisabled, Serial: true},
}
var (
defaultRetry = 3
)
const (
javascriptContainerTTLSeconds = 300
javascriptDatabaseReadyTimeout = 3 * time.Minute
javascriptServiceReadyTimeout = 45 * time.Second
javascriptRetryDelay = 500 * time.Millisecond
javascriptDialTimeout = 500 * time.Millisecond
)
type javascriptDockerReadyCheck func(address string) error
type javascriptDockerSpec struct {
options *dockertest.RunOptions
port string
readyTimeout time.Duration
settleDelay time.Duration
readyCheck javascriptDockerReadyCheck
}
func newJavascriptDockerSpec(port string, options *dockertest.RunOptions, readyTimeout time.Duration, settleDelay time.Duration, readyCheck javascriptDockerReadyCheck) javascriptDockerSpec {
if options == nil {
options = &dockertest.RunOptions{}
}
if options.Platform == "" {
options.Platform = "linux/amd64"
}
if len(options.ExposedPorts) == 0 {
options.ExposedPorts = []string{port}
}
if readyTimeout <= 0 {
readyTimeout = javascriptServiceReadyTimeout
}
return javascriptDockerSpec{
options: options,
port: port,
readyTimeout: readyTimeout,
settleDelay: settleDelay,
readyCheck: readyCheck,
}
}
type javascriptNetHttps struct{}
func (j *javascriptNetHttps) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptRedisPassBrute struct{}
func (j *javascriptRedisPassBrute) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("6379/tcp", &dockertest.RunOptions{
Repository: "redis",
Tag: "latest",
Cmd: []string{"redis-server", "--requirepass", "iamadmin"},
}, javascriptServiceReadyTimeout, 0, nil))
}
type javascriptSSHServerFingerprint struct{}
func (j *javascriptSSHServerFingerprint) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("22/tcp", &dockertest.RunOptions{
Repository: "alpine",
Tag: "latest",
Cmd: []string{
"sh",
"-c",
"apk add --no-cache openssh && ssh-keygen -A && mkdir -p /run/sshd && exec /usr/sbin/sshd -D -e",
},
}, javascriptServiceReadyTimeout, 2*time.Second, nil))
}
type javascriptVncPassBrute struct{}
func (j *javascriptVncPassBrute) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("5900/tcp", &dockertest.RunOptions{
Repository: "dorowu/ubuntu-desktop-lxde-vnc",
Tag: "latest",
Env: []string{
"VNC_PASSWORD=mysecret",
},
}, javascriptServiceReadyTimeout, 20*time.Second, nil))
}
type javascriptPostgresPassBrute struct{}
func (j *javascriptPostgresPassBrute) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("5432/tcp", &dockertest.RunOptions{
Repository: "postgres",
Tag: "13",
Env: []string{
"POSTGRES_INITDB_ARGS=--auth-host=md5 --auth-local=trust",
"POSTGRES_PASSWORD=postgres",
"POSTGRES_USER=postgres",
},
}, javascriptDatabaseReadyTimeout, 0, postgresReadyCheck("postgres", "postgres")), 1, 2)
}
type javascriptMySQLConnect struct{}
func (j *javascriptMySQLConnect) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("3306/tcp", &dockertest.RunOptions{
Repository: "mysql",
Tag: "5.7",
Env: []string{
"MYSQL_ROOT_PASSWORD=secret",
},
}, javascriptDatabaseReadyTimeout, 0, mysqlReadyCheck("root", "secret")))
}
type javascriptMultiPortsSSH struct{}
func (j *javascriptMultiPortsSSH) Execute(filePath string) error {
// use scanme.sh as target to ensure we match on the 2nd default port 22
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptNoPortArgs struct{}
func (j *javascriptNoPortArgs) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "yo.dawg", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type javascriptRsyncTest struct{}
func (j *javascriptRsyncTest) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("873/tcp", &dockertest.RunOptions{
Repository: "alpine",
Tag: "latest",
Cmd: []string{"sh", "-c", "apk add --no-cache rsync shadow && useradd -m rsyncuser && echo 'rsyncuser:mysecret' | chpasswd && echo 'rsyncuser:MySecret123' > /etc/rsyncd.secrets && chmod 600 /etc/rsyncd.secrets && echo -e '[data]\\n path = /data\\n comment = Local Rsync Share\\n read only = false\\n auth users = rsyncuser\\n secrets file = /etc/rsyncd.secrets' > /etc/rsyncd.conf && mkdir -p /data && exec rsync --daemon --no-detach --config=/etc/rsyncd.conf"},
}, javascriptServiceReadyTimeout, 0, nil))
}
type javascriptTelnetAuthTest struct{}
func (j *javascriptTelnetAuthTest) Execute(filePath string) error {
return runJavascriptDockerCase(filePath, newJavascriptDockerSpec("23/tcp", &dockertest.RunOptions{
Repository: "alpine",
Tag: "latest",
Cmd: []string{"sh", "-c", "apk add --no-cache busybox-extras shadow && useradd -m dev && echo 'dev:mysecret' | chpasswd && exec /usr/sbin/telnetd -F -p 23 -l /bin/login"},
}, javascriptServiceReadyTimeout, 0, nil))
}
type networkMultiStep struct{}
func (j *networkMultiStep) Execute(filePath string) error {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "FIRST" {
_, _ = conn.Write([]byte("PING"))
}
data, err = reader.ConnReadNWithTimeout(conn, 6, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "SECOND" {
_, _ = conn.Write([]byte("PONG"))
}
_, _ = conn.Write([]byte("NUCLEI"))
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, server.URL, debug)
if err != nil {
return err
}
if err := errState.Err(); err != nil {
return err
}
if debug {
return expectResultsCount(results, 3)
}
return expectResultsCount(results, 1)
}
func runJavascriptDockerCase(filePath string, spec javascriptDockerSpec, expectedNumbers ...int) error {
if len(expectedNumbers) == 0 {
expectedNumbers = []int{1}
}
if spec.options == nil {
return fmt.Errorf("missing docker options for %s", filePath)
}
pool, err := dockertest.NewPool("")
if err != nil {
return fmt.Errorf("could not create docker pool: %w", err)
}
if err := pool.Client.Ping(); err != nil {
return fmt.Errorf("could not connect to Docker: %w", err)
}
resource, err := pool.RunWithOptions(spec.options)
if err != nil {
return fmt.Errorf("could not start resource for %s: %w", filePath, err)
}
defer purge(pool, resource)
if err := resource.Expire(javascriptContainerTTLSeconds); err != nil {
return fmt.Errorf("could not expire resource for %s: %w", filePath, err)
}
mappedPort := resource.GetPort(spec.port)
if mappedPort == "" {
return fmt.Errorf("missing mapped port for %s", spec.port)
}
targetAddress := net.JoinHostPort("127.0.0.1", mappedPort)
if err := waitForTCPService(targetAddress, spec.readyTimeout); err != nil {
return err
}
if spec.readyCheck != nil {
if err := waitForServiceCheck(targetAddress, spec.readyTimeout, spec.readyCheck); err != nil {
return err
}
}
if spec.settleDelay > 0 {
time.Sleep(spec.settleDelay)
}
errS := make([]error, 0, defaultRetry)
for attempt := 1; attempt <= defaultRetry; attempt++ {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, targetAddress, debug)
if err == nil {
if countErr := expectResultsCount(results, expectedNumbers...); countErr == nil {
return nil
} else {
err = countErr
}
}
errS = append(errS, err)
if attempt < defaultRetry {
time.Sleep(javascriptRetryDelay)
}
}
return multierr.Combine(errS...)
}
func waitForTCPService(address string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", address, javascriptDialTimeout)
if err == nil {
_ = conn.Close()
return nil
}
lastErr = err
time.Sleep(javascriptRetryDelay)
}
if lastErr == nil {
lastErr = fmt.Errorf("timed out waiting for %s", address)
}
return fmt.Errorf("service %s did not become ready: %w", address, lastErr)
}
func waitForServiceCheck(address string, timeout time.Duration, check javascriptDockerReadyCheck) error {
deadline := time.Now().Add(timeout)
var lastErr error
for time.Now().Before(deadline) {
if err := check(address); err == nil {
return nil
} else {
lastErr = err
}
time.Sleep(javascriptRetryDelay)
}
if lastErr == nil {
lastErr = fmt.Errorf("timed out waiting for %s", address)
}
return fmt.Errorf("service %s did not become fully ready: %w", address, lastErr)
}
func mysqlReadyCheck(username, password string) javascriptDockerReadyCheck {
return func(address string) error {
dsn := fmt.Sprintf("%s:%s@tcp(%s)/information_schema?timeout=5s&readTimeout=5s&writeTimeout=5s", username, password, address)
db, err := sql.Open("mysql", dsn)
if err != nil {
return err
}
defer func() { _ = db.Close() }()
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(0)
return db.Ping()
}
}
func postgresReadyCheck(username, password string) javascriptDockerReadyCheck {
return func(address string) error {
db := pg.Connect(&pg.Options{
Addr: address,
User: username,
Password: password,
Database: "postgres",
}).WithTimeout(5 * time.Second)
defer func() { _ = db.Close() }()
_, err := db.Exec("select 1")
return err
}
}
func purge(pool *dockertest.Pool, resource *dockertest.Resource) {
if resource == nil || pool == nil {
return
}
containerName := resource.Container.Name
_ = pool.Client.StopContainer(resource.Container.ID, 0)
_ = pool.Purge(resource)
_ = pool.RemoveContainerByName(containerName)
}
cmd/integration-test/library.go → internal/tests/integration/library_test.go
+13 -8
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"context"
@@ -6,8 +9,7 @@ import (
"log"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
"time"
@@ -16,6 +18,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
@@ -30,12 +33,11 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/templates"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/ratelimit"
)
var libraryTestcases = []TestCaseInfo{
var libraryTestcases = []integrationCase{
{Path: "library/test.yaml", TestCase: &goIntegrationTest{}},
{Path: "library/test.json", TestCase: &goIntegrationTest{}},
}
@@ -85,7 +87,11 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
defer protocolstate.Close(defaultOpts.ExecutionId)
defaultOpts.Templates = goflags.StringSlice{templatePath}
actualTemplatePath := templatePath
if !filepath.IsAbs(actualTemplatePath) {
actualTemplatePath = fixturePath(actualTemplatePath)
}
defaultOpts.Templates = goflags.StringSlice{actualTemplatePath}
defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
outputWriter := testutils.NewMockOutputWriter(defaultOpts.OmitTemplate)
@@ -101,8 +107,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
}
defer interactClient.Close()
home, _ := os.UserHomeDir()
catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
catalog := disk.NewCatalog(suite.fixturesDir)
ratelimiter := ratelimit.New(context.Background(), 150, time.Second)
defer ratelimiter.Stop()
internal/tests/integration/loader_test.go
+166
View File
@@ -0,0 +1,166 @@
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/utils/errkit"
permissionutil "github.com/projectdiscovery/utils/permission"
)
func TestLoader(t *testing.T) {
t.Run("RemoteTemplateList", func(t *testing.T) {
server := newLoaderServer(t, "/template_list", fixturePath("loader/template-list.yaml"))
configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String())
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-template-url", server.URL+"/template_list",
"-config", configPath,
)
if err != nil {
t.Fatalf("expected remote template list to succeed: %v", err)
}
if err := expectResultsCount(results, 2); err != nil {
t.Fatal(err)
}
})
t.Run("ExcludedTemplate", func(t *testing.T) {
server := newLoaderServer(t, "", "")
templatePath := fixturePath("loader/excluded-template.yaml")
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-t", templatePath,
"-include-templates", templatePath,
)
if err != nil {
t.Fatalf("expected excluded template case to succeed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("RemoteTemplateListNotAllowed", func(t *testing.T) {
server := newLoaderServer(t, "/template_list", fixturePath("loader/template-list.yaml"))
_, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-template-url", server.URL+"/template_list",
)
if err == nil {
t.Fatal("expected error for remote template list without allow-list config")
}
})
t.Run("RemoteWorkflowList", func(t *testing.T) {
server := newLoaderServer(t, "/workflow_list", fixturePath("loader/workflow-list.yaml"))
configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String())
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-workflow-url", server.URL+"/workflow_list",
"-config", configPath,
)
if err != nil {
t.Fatalf("expected remote workflow list to succeed: %v", err)
}
if err := expectResultsCount(results, 3); err != nil {
t.Fatal(err)
}
})
t.Run("NonExistentTemplateList", func(t *testing.T) {
server := newLoaderServer(t, "", "")
configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String())
_, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-template-url", server.URL+"/404",
"-config", configPath,
)
if err == nil {
t.Fatal("expected error for non-existing remote template list")
}
})
t.Run("NonExistentWorkflowList", func(t *testing.T) {
server := newLoaderServer(t, "", "")
configPath := writeAllowedRemoteConfig(t, server.Listener.Addr().String())
_, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, nil,
"-target", server.URL,
"-workflow-url", server.URL+"/404",
"-config", configPath,
)
if err == nil {
t.Fatal("expected error for non-existing remote workflow list")
}
})
t.Run("LoadTemplateWithID", func(t *testing.T) {
server := newLoaderServer(t, "", "")
templateDir := t.TempDir()
copyFixtureToDir(t, "library/test.yaml", templateDir)
results, err := testutils.RunNucleiBareArgsAndGetResults(suite.debug, []string{"NUCLEI_TEMPLATES_DIR=" + templateDir},
"-target", server.URL,
"-id", "go-integration-test",
)
if err != nil {
t.Fatalf("failed to load template with id: %v", errkit.Wrap(err, "load template with id"))
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
func newLoaderServer(t *testing.T, listPath, listFixture string) *httptest.Server {
t.Helper()
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = fmt.Fprint(w, "This is test matcher text")
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
_, _ = fmt.Fprint(w, "This is test headers matcher text")
}
})
if listPath != "" {
router.GET(listPath, func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
file, err := os.ReadFile(listFixture)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, _ = w.Write(file)
})
}
server := httptest.NewServer(router)
t.Cleanup(server.Close)
return server
}
func writeAllowedRemoteConfig(t *testing.T, allowedAddress string) string {
t.Helper()
configPath := filepath.Join(t.TempDir(), "test-config.yaml")
configData := []byte(`remote-template-domain: [ "` + allowedAddress + `" ]`)
if err := os.WriteFile(configPath, configData, permissionutil.ConfigFilePermission); err != nil {
t.Fatalf("failed to write remote template allow-list config: %v", err)
}
return configPath
}
cmd/integration-test/matcher-status.go → internal/tests/integration/matcher-status_test.go
+6 -3
View File
@@ -1,4 +1,7 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
@@ -7,12 +10,12 @@ import (
"strings"
"github.com/julienschmidt/httprouter"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
)
var matcherStatusTestcases = []TestCaseInfo{
var matcherStatusTestcases = []integrationCase{
{Path: "protocols/http/get.yaml", TestCase: &httpNoAccess{}},
{Path: "protocols/network/net-https.yaml", TestCase: &networkNoAccess{}},
{Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessNoAccess{}},
cmd/integration-test/multi.go → internal/tests/integration/multi_test.go
+6 -3
View File
@@ -1,10 +1,13 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
var multiProtoTestcases = []TestCaseInfo{
var multiProtoTestcases = []integrationCase{
{Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}},
{Path: "protocols/multi/evaluate-variables.yaml", TestCase: &multiProtoDynamicExtractor{}},
{Path: "protocols/multi/exported-response-vars.yaml", TestCase: &multiProtoDynamicExtractor{}},
internal/tests/integration/network_test.go
+262
View File
@@ -0,0 +1,262 @@
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"net"
"strings"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/utils/reader"
)
func TestNetwork(t *testing.T) {
t.Run("Basic", func(t *testing.T) {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
return
}
errState.Set(fmt.Errorf("invalid data received: %s", string(data)))
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/basic.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("basic network request failed: %v", err)
}
if err := errState.Err(); err != nil {
t.Fatal(err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("Hex", func(t *testing.T) {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
return
}
errState.Set(fmt.Errorf("invalid data received: %s", string(data)))
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/hex.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("hex network request failed: %v", err)
}
if err := errState.Err(); err != nil {
t.Fatal(err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("MultiStep", func(t *testing.T) {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 5, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "FIRST" {
_, _ = conn.Write([]byte("PING"))
}
data, err = reader.ConnReadNWithTimeout(conn, 6, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "SECOND" {
_, _ = conn.Write([]byte("PONG"))
}
_, _ = conn.Write([]byte("NUCLEI"))
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/multi-step.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("multi-step network request failed: %v", err)
}
if err := errState.Err(); err != nil {
t.Fatal(err)
}
expectedCount := 1
if suite.debug {
expectedCount = 3
}
if err := expectResultsCount(results, expectedCount); err != nil {
t.Fatal(err)
}
})
t.Run("SelfContained", func(t *testing.T) {
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
_, _ = conn.Write([]byte("Authentication successful"))
})
defer server.Close()
templatePath := tempFixtureCopy(t, "protocols/network/self-contained.yaml", map[string]string{
"127.0.0.1:5431": server.URL,
})
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "", suite.debug, "-esc")
if err != nil {
t.Fatalf("self-contained network request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("Variables", func(t *testing.T) {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("aGVsbG8="))
}
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/variables.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("variables network request failed: %v", err)
}
if err := errState.Err(); err != nil {
t.Fatal(err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("SameAddress", func(t *testing.T) {
errState := &capturedError{}
server := testutils.NewTCPServer(nil, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err != nil {
errState.Set(err)
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
return
}
errState.Set(fmt.Errorf("invalid data received: %s", string(data)))
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/same-address.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("same-address network request failed: %v", err)
}
if err := errState.Err(); err != nil {
t.Fatal(err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("NetworkPort", func(t *testing.T) {
server := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err == nil && string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
defer server.Close()
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("network-port template failed with template port: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
results, err = testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", strings.ReplaceAll(server.URL, "23846", "443"), suite.debug)
if err != nil {
t.Fatalf("network-port template failed with overridden input port: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
serverOverride := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data, err := reader.ConnReadNWithTimeout(conn, 4, 5*time.Second)
if err == nil && string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
defer serverOverride.Close()
results, err = testutils.RunNucleiTemplateAndGetResults("protocols/network/network-port.yaml", serverOverride.URL, suite.debug)
if err != nil {
t.Fatalf("network-port template failed with runtime override port: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("NetHTTPS", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/net-https.yaml", "scanme.sh", suite.debug)
if err != nil {
t.Fatalf("network https request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("NetHTTPSTimeout", func(t *testing.T) {
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/network/net-https-timeout.yaml", "scanme.sh", suite.debug)
if err != nil {
t.Fatalf("network https timeout request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
}
cmd/integration-test/offline-http.go → internal/tests/integration/offline-http_test.go
+6 -3
View File
@@ -1,12 +1,15 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
var offlineHttpTestcases = []TestCaseInfo{
var offlineHttpTestcases = []integrationCase{
{Path: "protocols/offlinehttp/rfc-req-resp.yaml", TestCase: &RfcRequestResponse{}},
{Path: "protocols/offlinehttp/offline-allowed-paths.yaml", TestCase: &RequestResponseWithAllowedPaths{}},
{Path: "protocols/offlinehttp/offline-raw.yaml", TestCase: &RawRequestResponse{}},
cmd/integration-test/profile-loader.go → internal/tests/integration/profile-loader_test.go
+6 -3
View File
@@ -1,13 +1,16 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/utils/errkit"
)
var profileLoaderTestcases = []TestCaseInfo{
var profileLoaderTestcases = []integrationCase{
{Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}},
{Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}},
{Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}},
internal/tests/integration/runner_test.go
+300
View File
@@ -0,0 +1,300 @@
//go:build integration
// +build integration
package integration_test
import (
"flag"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
type integrationCase struct {
Path string
TestCase testutils.TestCase
DisableOn func() bool
Serial bool
}
type integrationFamily struct {
Name string
Cases []integrationCase
Serial bool
}
type integrationFailures struct {
mu sync.Mutex
items []failedIntegrationCase
}
type failedIntegrationCase struct {
Family string
Case integrationCase
Err error
}
type stringSliceFlag []string
func (s *stringSliceFlag) String() string {
return strings.Join(*s, " ")
}
func (s *stringSliceFlag) Set(value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil
}
*s = append(*s, trimmed)
return nil
}
var (
integrationFamilyFilter = flag.String("family", "", "run only the matching integration family")
integrationTemplateFilter = flag.String("template", "", "run only integration cases whose template path contains this value")
integrationExtraArgs stringSliceFlag
)
const maxFailureReruns = 3
func init() {
flag.Var(&integrationExtraArgs, "nuclei-args", "extra argument passed to nuclei for integration tests; repeat the flag to pass multiple arguments")
}
func TestIntegrationSuites(t *testing.T) {
families := selectedIntegrationFamilies()
if len(families) == 0 {
t.Fatalf("no integration tests matched family=%q template=%q", strings.TrimSpace(*integrationFamilyFilter), strings.TrimSpace(*integrationTemplateFilter))
}
failed := runIntegrationPass(t, "initial", families, debug)
if len(failed) == 0 {
return
}
if isCI() {
for attempt := 1; attempt <= maxFailureReruns && len(failed) > 0; attempt++ {
failed = withGroupedOutput(fmt.Sprintf("Rerun failed integration tests (%d/%d)", attempt, maxFailureReruns), func() []failedIntegrationCase {
return runIntegrationPass(t, fmt.Sprintf("rerun-%d", attempt), failedCasesByFamily(failed), debug)
})
}
}
if len(failed) > 0 && shouldDebugFailureRerun() {
failed = withGroupedOutput("Rerun failed integration tests in debug mode", func() []failedIntegrationCase {
return runIntegrationPass(t, "debug-rerun", failedCasesByFamily(failed), true)
})
}
if len(failed) > 0 {
t.Fatalf("integration failures remaining after reruns:\n%s", formatIntegrationFailures(failed))
}
}
func integrationFamilies() []integrationFamily {
return []integrationFamily{
{Name: "http", Cases: httpTestcases},
{Name: "interactsh", Cases: interactshTestCases},
{Name: "code", Cases: codeTestCases},
{Name: "workflow", Cases: workflowTestcases},
{Name: "headless", Cases: headlessTestcases},
{Name: "whois", Cases: whoisTestCases},
{Name: "library", Cases: libraryTestcases},
{Name: "template-path", Cases: templatesPathTestCases},
{Name: "offline-http", Cases: offlineHttpTestcases},
{Name: "fuzz", Cases: fuzzingTestCases},
{Name: "generic", Cases: genericTestcases},
{Name: "dsl", Cases: dslTestcases},
{Name: "javascript", Cases: jsTestcases},
{Name: "matcher-status", Cases: matcherStatusTestcases},
{Name: "exporters", Cases: exportersTestCases},
{Name: "profile-loader", Cases: profileLoaderTestcases},
{Name: "multi-protocol", Cases: multiProtoTestcases},
}
}
func runIntegrationPass(t *testing.T, label string, families []integrationFamily, debugMode bool) []failedIntegrationCase {
semaphore := make(chan struct{}, parallelism())
failures := &integrationFailures{}
previousDebug := debug
debug = debugMode
defer func() {
debug = previousDebug
}()
for _, family := range families {
family := family
t.Run(passFamilyName(label, family.Name), func(t *testing.T) {
runIntegrationFamily(t, semaphore, failures, family)
})
}
return failures.Items()
}
func runIntegrationFamily(t *testing.T, semaphore chan struct{}, failures *integrationFailures, family integrationFamily) {
t.Helper()
for _, testCase := range family.Cases {
testCase := testCase
t.Run(testNameForPath(testCase.Path), func(t *testing.T) {
if !family.Serial && !testCase.Serial {
t.Parallel()
semaphore <- struct{}{}
defer func() { <-semaphore }()
}
if err := executeIntegrationCase(testCase); err != nil {
failures.Add(failedIntegrationCase{Family: family.Name, Case: testCase, Err: err})
t.Logf("integration case failed: %v", err)
}
})
}
}
func executeIntegrationCase(testCase integrationCase) error {
if testCase.DisableOn != nil && testCase.DisableOn() {
return nil
}
if needsSignedCodeTemplates(testCase.Path) {
if err := ensureSignedCodeTemplates(); err != nil {
return fmt.Errorf("failed to sign code templates: %w", err)
}
}
var retries int
var err error
for attempt := 1; attempt <= maxFailureReruns; attempt++ {
retries++
err = testCase.TestCase.Execute(testCase.Path)
if err == nil {
return nil
}
}
if retries > 1 {
return fmt.Errorf("test %s failed after %d attempts: %w", testCase.Path, retries, err)
}
return err
}
func parallelism() int {
if value := strings.TrimSpace(os.Getenv("PARALLELISM")); value != "" {
if parsed, err := strconv.Atoi(value); err == nil && parsed > 0 {
return parsed
}
}
parallelism := max(min(runtime.GOMAXPROCS(0), 4), 1)
return parallelism
}
func needsSignedCodeTemplates(path string) bool {
return strings.HasPrefix(path, "protocols/code/") || strings.Contains(path, "workflow/code-")
}
func testNameForPath(path string) string {
replacer := strings.NewReplacer("/", "_", "\\", "_", ",", "__", ".yaml", "", ".yml", "", ".json", "", ".", "_")
name := replacer.Replace(path)
name = strings.Trim(name, "_")
if name == "" {
return "case"
}
return fmt.Sprintf("%s", name)
}
func (f *integrationFailures) Add(failure failedIntegrationCase) {
if failure.Err == nil {
return
}
f.mu.Lock()
defer f.mu.Unlock()
f.items = append(f.items, failure)
}
func (f *integrationFailures) Items() []failedIntegrationCase {
f.mu.Lock()
defer f.mu.Unlock()
items := make([]failedIntegrationCase, len(f.items))
copy(items, f.items)
return items
}
func selectedIntegrationFamilies() []integrationFamily {
familyFilter := strings.ToLower(strings.TrimSpace(*integrationFamilyFilter))
templateFilter := strings.ToLower(strings.TrimSpace(*integrationTemplateFilter))
selected := make([]integrationFamily, 0)
for _, family := range integrationFamilies() {
if familyFilter != "" && !strings.EqualFold(family.Name, familyFilter) {
continue
}
if templateFilter == "" {
selected = append(selected, family)
continue
}
filteredCases := make([]integrationCase, 0, len(family.Cases))
for _, testCase := range family.Cases {
if strings.Contains(strings.ToLower(testCase.Path), templateFilter) {
filteredCases = append(filteredCases, testCase)
}
}
if len(filteredCases) == 0 {
continue
}
family.Cases = filteredCases
selected = append(selected, family)
}
return selected
}
func failedCasesByFamily(failed []failedIntegrationCase) []integrationFamily {
orderedFamilies := make([]string, 0)
byFamily := make(map[string][]integrationCase)
for _, failure := range failed {
if _, ok := byFamily[failure.Family]; !ok {
orderedFamilies = append(orderedFamilies, failure.Family)
}
byFamily[failure.Family] = append(byFamily[failure.Family], failure.Case)
}
families := make([]integrationFamily, 0, len(orderedFamilies))
for _, familyName := range orderedFamilies {
serial := false
for _, family := range integrationFamilies() {
if family.Name == familyName {
serial = family.Serial
break
}
}
families = append(families, integrationFamily{Name: familyName, Cases: byFamily[familyName], Serial: serial})
}
return families
}
func passFamilyName(label, family string) string {
if label == "" || label == "initial" {
return family
}
return label + "/" + family
}
func shouldDebugFailureRerun() bool {
return envTruthy("DEBUG") || envTruthy("ACTIONS_STEP_DEBUG")
}
func withGroupedOutput[T any](title string, fn func() T) T {
if isCI() {
fmt.Printf("::group::%s\n", title)
defer fmt.Println("::endgroup::")
}
return fn()
}
func formatIntegrationFailures(failed []failedIntegrationCase) string {
lines := make([]string, 0, len(failed))
for _, failure := range failed {
lines = append(lines, fmt.Sprintf("- [%s] %s: %v", failure.Family, failure.Case.Path, failure.Err))
}
return strings.Join(lines, "\n")
}
internal/tests/integration/ssl_test.go
+95
View File
@@ -0,0 +1,95 @@
//go:build integration
// +build integration
package integration_test
import (
"crypto/tls"
"net"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
)
func TestSSL(t *testing.T) {
t.Run("Basic", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/basic.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("basic ssl request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("BasicZTLS", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/basic-ztls.yaml", server.URL, suite.debug, "-ztls")
if err != nil {
t.Fatalf("basic ztls request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("CustomCipher", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{CipherSuites: []uint16{tls.TLS_AES_128_GCM_SHA256}})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/custom-cipher.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("custom cipher ssl request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("CustomVersion", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{MinVersion: tls.VersionTLS12, MaxVersion: tls.VersionTLS12})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/custom-version.yaml", server.URL, suite.debug)
if err != nil {
t.Fatalf("custom version ssl request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("WithVars", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/ssl-with-vars.yaml", server.URL, suite.debug, "-V", "test=asdasdas")
if err != nil {
t.Fatalf("ssl with vars request failed: %v", err)
}
if err := expectResultsCount(results, 1); err != nil {
t.Fatal(err)
}
})
t.Run("MultiReq", func(t *testing.T) {
server := newTLSServer(t, &tls.Config{
//nolint:staticcheck // SSLv3 is intentionally used for testing purposes.
MinVersion: tls.VersionSSL30,
MaxVersion: tls.VersionTLS11,
})
results, err := testutils.RunNucleiTemplateAndGetResults("protocols/ssl/multi-req.yaml", server.URL, suite.debug, "-V")
if err != nil {
t.Fatalf("ssl multi request failed: %v", err)
}
if err := expectResultsCount(results, 2); err != nil {
t.Fatal(err)
}
})
}
func newTLSServer(t *testing.T, config *tls.Config) *testutils.TCPServer {
t.Helper()
server := testutils.NewTCPServer(config, 0, func(conn net.Conn) {
defer func() { _ = conn.Close() }()
data := make([]byte, 4)
_, _ = conn.Read(data)
})
t.Cleanup(server.Close)
return server
}
cmd/integration-test/template-path.go → internal/tests/integration/template-path_test.go
+6 -3
View File
@@ -1,18 +1,21 @@
package main
//go:build integration
// +build integration
package integration_test
import (
"fmt"
"strings"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
)
func getTemplatePath() string {
return config.DefaultConfig.TemplatesDirectory
}
var templatesPathTestCases = []TestCaseInfo{
var templatesPathTestCases = []integrationCase{
//template folder path issue
{Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}},
//cwd
integration_tests/dsl/hide-version-warning.yaml → internal/tests/integration/testdata/dsl/hide-version-warning.yaml
Vendored
View File
integration_tests/dsl/show-version-warning.yaml → internal/tests/integration/testdata/dsl/show-version-warning.yaml
Vendored
View File
integration_tests/flow/conditional-flow-negative.yaml → internal/tests/integration/testdata/flow/conditional-flow-negative.yaml
Vendored
View File
integration_tests/flow/conditional-flow.yaml → internal/tests/integration/testdata/flow/conditional-flow.yaml
Vendored
View File
integration_tests/flow/dns-ns-probe.yaml → internal/tests/integration/testdata/flow/dns-ns-probe.yaml
Vendored
View File
integration_tests/flow/flow-hide-matcher.yaml → internal/tests/integration/testdata/flow/flow-hide-matcher.yaml
Vendored
View File
integration_tests/flow/iterate-one-value-flow.yaml → internal/tests/integration/testdata/flow/iterate-one-value-flow.yaml
Vendored
View File
integration_tests/flow/iterate-values-flow.yaml → internal/tests/integration/testdata/flow/iterate-values-flow.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body-generic-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-body-generic-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body-json-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-body-json-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-body-multipart-form-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body-params-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-body-params-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body-xml-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-body-xml-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-body.yaml → internal/tests/integration/testdata/fuzz/fuzz-body.yaml
Vendored
View File
integration_tests/fuzz/fuzz-cookie-error-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-cookie-error-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-headless.yaml → internal/tests/integration/testdata/fuzz/fuzz-headless.yaml
Vendored
View File
integration_tests/fuzz/fuzz-host-header-injection.yaml → internal/tests/integration/testdata/fuzz/fuzz-host-header-injection.yaml
Vendored
View File
integration_tests/fuzz/fuzz-mode.yaml → internal/tests/integration/testdata/fuzz/fuzz-mode.yaml
Vendored
View File
integration_tests/fuzz/fuzz-multi-mode.yaml → internal/tests/integration/testdata/fuzz/fuzz-multi-mode.yaml
Vendored
View File
integration_tests/fuzz/fuzz-path-sqli.yaml → internal/tests/integration/testdata/fuzz/fuzz-path-sqli.yaml
Vendored
View File
integration_tests/fuzz/fuzz-query-num-replace.yaml → internal/tests/integration/testdata/fuzz/fuzz-query-num-replace.yaml
Vendored
View File
integration_tests/fuzz/fuzz-query.yaml → internal/tests/integration/testdata/fuzz/fuzz-query.yaml
Vendored
View File
integration_tests/fuzz/fuzz-type.yaml → internal/tests/integration/testdata/fuzz/fuzz-type.yaml
Vendored
View File
integration_tests/fuzz/testData/ginandjuice.proxify.yaml → internal/tests/integration/testdata/fuzz/testData/ginandjuice.proxify.yaml
Vendored
View File
integration_tests/generic/auth/certificate/assets/client.crt → internal/tests/integration/testdata/generic/auth/certificate/assets/client.crt
Vendored
View File
integration_tests/generic/auth/certificate/assets/client.key → internal/tests/integration/testdata/generic/auth/certificate/assets/client.key
Vendored
View File
integration_tests/generic/auth/certificate/assets/server.crt → internal/tests/integration/testdata/generic/auth/certificate/assets/server.crt
Vendored
View File
integration_tests/generic/auth/certificate/http-get.yaml → internal/tests/integration/testdata/generic/auth/certificate/http-get.yaml
Vendored
View File
integration_tests/library/test.json → internal/tests/integration/testdata/library/test.json
Vendored
View File
integration_tests/library/test.yaml → internal/tests/integration/testdata/library/test.yaml
Vendored
View File
integration_tests/loader/basic.yaml → internal/tests/integration/testdata/loader/basic.yaml
Vendored
View File
integration_tests/loader/condition-matched.yaml → internal/tests/integration/testdata/loader/condition-matched.yaml
Vendored
View File
integration_tests/loader/excluded-template.yaml → internal/tests/integration/testdata/loader/excluded-template.yaml
Vendored
View File

Some files were not shown because too many files have changed in this diff Show More