Merge branch 'dev' into feature/secret-file-auth

README.md

@@ -92,9 +92,10 @@ Usage:
 
 Flags:
 INPUT:
-   -l, -list string      input file containing list of hosts to process
-   -rr, -request string  file containing raw request
-   -u, -target string[]  input target host(s) to probe
+   -l, -list string              input file containing list of hosts to process
+   -rr, -request string          file containing raw request
+   -u, -target string[]          input target host(s) to probe
+   -im, -input-mode string       mode of input file (burp)
 
 PROBES:
    -sc, -status-code      display response status-code
@@ -279,6 +280,7 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
 # Notes
 
 - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
+- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
 - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
 - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)

common/inputformats/burp.go

@@ -0,0 +1,44 @@
+package inputformats
+
+import (
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/seh-msft/burpxml"
+)
+
+// BurpFormat is a Burp Suite XML file parser
+type BurpFormat struct{}
+
+// NewBurpFormat creates a new Burp XML file parser
+func NewBurpFormat() *BurpFormat {
+	return &BurpFormat{}
+}
+
+var _ Format = &BurpFormat{}
+
+// Name returns the name of the format
+func (b *BurpFormat) Name() string {
+	return "burp"
+}
+
+// Parse parses the Burp XML input and calls the provided callback
+// function for each URL it discovers.
+func (b *BurpFormat) Parse(input io.Reader, callback func(url string) bool) error {
+	items, err := burpxml.Parse(input, true)
+	if err != nil {
+		return errors.Wrap(err, "could not parse burp xml")
+	}
+
+	for i, item := range items.Items {
+		if item.Url == "" {
+			gologger.Debug().Msgf("Skipping burp item %d: empty URL", i)
+			continue
+		}
+		if !callback(item.Url) {
+			break
+		}
+	}
+	return nil
+}

common/inputformats/burp_test.go

@@ -0,0 +1,169 @@
+package inputformats
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestBurpFormat_Name(t *testing.T) {
+	b := NewBurpFormat()
+	if b.Name() != "burp" {
+		t.Errorf("Expected name 'burp', got '%s'", b.Name())
+	}
+}
+
+func TestBurpFormat_Parse(t *testing.T) {
+	burpXML := `<?xml version="1.0"?>
+<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
+  <item>
+    <time>Sat Sep 30 20:11:32 IST 2023</time>
+    <url><![CDATA[http://example.com/path1]]></url>
+    <host ip="127.0.0.1">example.com</host>
+    <port>80</port>
+    <protocol>http</protocol>
+    <method><![CDATA[GET]]></method>
+    <path><![CDATA[/path1]]></path>
+    <extension>null</extension>
+    <request base64="true"><![CDATA[R0VUIC8gSFRUUC8xLjE=]]></request>
+    <status>200</status>
+    <responselength>100</responselength>
+    <mimetype>HTML</mimetype>
+    <response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
+    <comment></comment>
+  </item>
+  <item>
+    <time>Sat Sep 30 20:08:54 IST 2023</time>
+    <url><![CDATA[https://example.com/path2]]></url>
+    <host ip="127.0.0.1">example.com</host>
+    <port>443</port>
+    <protocol>https</protocol>
+    <method><![CDATA[POST]]></method>
+    <path><![CDATA[/path2]]></path>
+    <extension>null</extension>
+    <request base64="true"><![CDATA[UE9TVCAvIEhUVFAvMS4x]]></request>
+    <status>200</status>
+    <responselength>100</responselength>
+    <mimetype>JSON</mimetype>
+    <response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
+    <comment></comment>
+  </item>
+</items>`
+
+	b := NewBurpFormat()
+	var urls []string
+
+	err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
+		urls = append(urls, url)
+		return true
+	})
+
+	if err != nil {
+		t.Fatalf("Parse returned error: %v", err)
+	}
+
+	if len(urls) != 2 {
+		t.Errorf("Expected 2 URLs, got %d", len(urls))
+	}
+
+	expectedURLs := []string{"http://example.com/path1", "https://example.com/path2"}
+	if len(urls) != len(expectedURLs) {
+		t.Fatalf("Expected %d URLs, got %d: %v", len(expectedURLs), len(urls), urls)
+	}
+	for i, expected := range expectedURLs {
+		if urls[i] != expected {
+			t.Errorf("Expected URL %d to be '%s', got '%s'", i, expected, urls[i])
+		}
+	}
+}
+
+func TestBurpFormat_ParseEmpty(t *testing.T) {
+	burpXML := `<?xml version="1.0"?>
+<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
+</items>`
+
+	b := NewBurpFormat()
+	var urls []string
+
+	err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
+		urls = append(urls, url)
+		return true
+	})
+
+	if err != nil {
+		t.Fatalf("Parse returned error: %v", err)
+	}
+
+	if len(urls) != 0 {
+		t.Errorf("Expected 0 URLs, got %d", len(urls))
+	}
+}
+
+func TestBurpFormat_ParseStopEarly(t *testing.T) {
+	burpXML := `<?xml version="1.0"?>
+<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
+  <item>
+    <url><![CDATA[http://example.com/1]]></url>
+    <host>example.com</host>
+    <port>80</port>
+    <protocol>http</protocol>
+    <method><![CDATA[GET]]></method>
+    <path><![CDATA[/1]]></path>
+    <extension>null</extension>
+    <request base64="true"><![CDATA[R0VUIC8=]]></request>
+    <status>200</status>
+    <responselength>100</responselength>
+    <mimetype>HTML</mimetype>
+    <response base64="true"><![CDATA[T0s=]]></response>
+    <comment></comment>
+  </item>
+  <item>
+    <url><![CDATA[http://example.com/2]]></url>
+    <host>example.com</host>
+    <port>80</port>
+    <protocol>http</protocol>
+    <method><![CDATA[GET]]></method>
+    <path><![CDATA[/2]]></path>
+    <extension>null</extension>
+    <request base64="true"><![CDATA[R0VUIC8=]]></request>
+    <status>200</status>
+    <responselength>100</responselength>
+    <mimetype>HTML</mimetype>
+    <response base64="true"><![CDATA[T0s=]]></response>
+    <comment></comment>
+  </item>
+</items>`
+
+	b := NewBurpFormat()
+	var urls []string
+
+	err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
+		urls = append(urls, url)
+		return false // stop after first
+	})
+
+	if err != nil {
+		t.Fatalf("Parse returned error: %v", err)
+	}
+
+	if len(urls) != 1 {
+		t.Errorf("Expected 1 URL (stopped early), got %d", len(urls))
+	}
+}
+
+func TestBurpFormat_ParseMalformed(t *testing.T) {
+	malformedXML := `<?xml version="1.0"?>
+<items burpVersion="2023.10.1.2">
+  <item>
+    <url><![CDATA[http://example.com/path1]]></url>
+    <!-- missing closing tags -->
+</items>`
+
+	b := NewBurpFormat()
+	err := b.Parse(strings.NewReader(malformedXML), func(url string) bool {
+		return true
+	})
+
+	if err == nil {
+		t.Error("Expected error for malformed XML, got nil")
+	}
+}

common/inputformats/formats.go

@@ -0,0 +1,41 @@
+// TODO: This package should be abstracted out to projectdiscovery/utils
+// so it can be shared between httpx, nuclei, and other tools.
+package inputformats
+
+import (
+	"io"
+	"strings"
+)
+
+// Format is an interface implemented by all input formats
+type Format interface {
+	// Name returns the name of the format
+	Name() string
+	// Parse parses the input and calls the provided callback
+	// function for each URL it discovers.
+	Parse(input io.Reader, callback func(url string) bool) error
+}
+
+// Supported formats
+var formats = []Format{
+	NewBurpFormat(),
+}
+
+// GetFormat returns the format by name
+func GetFormat(name string) Format {
+	for _, f := range formats {
+		if strings.EqualFold(f.Name(), name) {
+			return f
+		}
+	}
+	return nil
+}
+
+// SupportedFormats returns a comma-separated list of supported format names
+func SupportedFormats() string {
+	var names []string
+	for _, f := range formats {
+		names = append(names, f.Name())
+	}
+	return strings.Join(names, ", ")
+}

common/inputformats/formats_test.go

@@ -0,0 +1,43 @@
+package inputformats
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestGetFormat(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		wantNil  bool
+		wantName string
+	}{
+		{"burp lowercase", "burp", false, "burp"},
+		{"burp uppercase", "BURP", false, "burp"},
+		{"burp mixed case", "Burp", false, "burp"},
+		{"invalid format", "invalid", true, ""},
+		{"empty string", "", true, ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := GetFormat(tt.input)
+			if tt.wantNil && got != nil {
+				t.Errorf("GetFormat(%q) = %v, want nil", tt.input, got)
+			}
+			if !tt.wantNil && got == nil {
+				t.Errorf("GetFormat(%q) = nil, want non-nil", tt.input)
+			}
+			if !tt.wantNil && got != nil && got.Name() != tt.wantName {
+				t.Errorf("GetFormat(%q).Name() = %q, want %q", tt.input, got.Name(), tt.wantName)
+			}
+		})
+	}
+}
+
+func TestSupportedFormats(t *testing.T) {
+	supported := SupportedFormats()
+	if !strings.Contains(supported, "burp") {
+		t.Errorf("SupportedFormats() = %q, expected to contain 'burp'", supported)
+	}
+}

go.mod

@@ -54,6 +54,8 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-viper/mapstructure/v2 v2.4.0
 	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
+	github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193
+	github.com/seh-msft/burpxml v1.0.1
 	github.com/weppos/publicsuffix-go v0.50.2
 )
 
@@ -128,7 +130,6 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.23 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/freeport v0.0.7 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect

go.sum

@@ -389,6 +389,8 @@ github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uR
 github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
 github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
+github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg=
+github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=

runner/options.go

@@ -23,6 +23,7 @@ import (
 	customport "github.com/projectdiscovery/httpx/common/customports"
 	fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
 	httpxcommon "github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/httpx/common/inputformats"
 	"github.com/projectdiscovery/httpx/common/stringz"
 	"github.com/projectdiscovery/networkpolicy"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -191,6 +192,7 @@ type Options struct {
 	SocksProxy                string
 	Proxy                     string
 	InputFile                 string
+	InputMode                 string
 	InputTargetHost           goflags.StringSlice
 	Methods                   string
 	RequestURI                string
@@ -377,6 +379,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
 		flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
 		flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringVarP(&options.InputMode, "input-mode", "im", "", fmt.Sprintf("mode of input file (%s)", inputformats.SupportedFormats())),
 	)
 
 	flagSet.CreateGroup("Probes", "Probes",
@@ -683,6 +686,13 @@ func (options *Options) ValidateOptions() error {
 	if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) {
 		return fmt.Errorf("secret file '%s' does not exist", options.SecretFile)
 	}
+	if options.InputMode != "" && inputformats.GetFormat(options.InputMode) == nil {
+		return fmt.Errorf("invalid input mode '%s', supported formats: %s", options.InputMode, inputformats.SupportedFormats())
+	}
+
+	if options.InputMode != "" && options.InputFile == "" {
+		return errors.New("-im/-input-mode requires -l/-list to specify an input file")
+	}
 
 	if options.Silent {
 		incompatibleFlagsList := flagsIncompatibleWithSilent(options)

runner/runner.go

@@ -35,6 +35,7 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/httpx/common/customextract"
 	"github.com/projectdiscovery/httpx/common/hashes/jarm"
+	"github.com/projectdiscovery/httpx/common/inputformats"
 	"github.com/projectdiscovery/httpx/common/pagetypeclassifier"
 	"github.com/projectdiscovery/httpx/common/authprovider"
 	"github.com/projectdiscovery/httpx/static"
@@ -536,13 +537,22 @@ func (r *Runner) prepareInput() {
 	}
 	// check if file has been provided
 	if fileutil.FileExists(r.options.InputFile) {
-		finput, err := os.Open(r.options.InputFile)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
-		}
-		numHosts, err = r.loadAndCloseFile(finput)
-		if err != nil {
-			gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
+		// check if input mode is specified for special format handling
+		if format := r.getInputFormat(); format != nil {
+			numTargets, err := r.loadFromFormat(r.options.InputFile, format)
+			if err != nil {
+				gologger.Fatal().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err)
+			}
+			numHosts = numTargets
+		} else {
+			finput, err := os.Open(r.options.InputFile)
+			if err != nil {
+				gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
+			}
+			numHosts, err = r.loadAndCloseFile(finput)
+			if err != nil {
+				gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
+			}
 		}
 	} else if r.options.InputFile != "" {
 		files, err := fileutilz.ListFilesWithPattern(r.options.InputFile)
@@ -636,19 +646,52 @@ func (r *Runner) testAndSet(k string) bool {
 	return true
 }
 
+// getInputFormat returns the format for the configured input mode.
+// Returns nil if no input mode is configured, or logs fatal if the format is invalid.
+func (r *Runner) getInputFormat() inputformats.Format {
+	if r.options.InputMode == "" {
+		return nil
+	}
+	format := inputformats.GetFormat(r.options.InputMode)
+	if format == nil {
+		gologger.Fatal().Msgf("Invalid input mode '%s'. Supported: %s\n", r.options.InputMode, inputformats.SupportedFormats())
+	}
+	return format
+}
+
 func (r *Runner) streamInput() (chan string, error) {
 	out := make(chan string)
 	go func() {
 		defer close(out)
 
 		if fileutil.FileExists(r.options.InputFile) {
-			fchan, err := fileutil.ReadFile(r.options.InputFile)
-			if err != nil {
-				return
-			}
-			for item := range fchan {
-				if r.options.SkipDedupe || r.testAndSet(item) {
-					out <- item
+			// check if input mode is specified for special format handling
+			if format := r.getInputFormat(); format != nil {
+				finput, err := os.Open(r.options.InputFile)
+				if err != nil {
+					gologger.Error().Msgf("Could not open input file '%s': %s\n", r.options.InputFile, err)
+					return
+				}
+				defer finput.Close()
+				if err := format.Parse(finput, func(item string) bool {
+					item = strings.TrimSpace(item)
+					if r.options.SkipDedupe || r.testAndSet(item) {
+						out <- item
+					}
+					return true
+				}); err != nil {
+					gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err)
+					return
+				}
+			} else {
+				fchan, err := fileutil.ReadFile(r.options.InputFile)
+				if err != nil {
+					return
+				}
+				for item := range fchan {
+					if r.options.SkipDedupe || r.testAndSet(item) {
+						out <- item
+					}
 				}
 			}
 		} else if r.options.InputFile != "" {
@@ -704,6 +747,31 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
 	return numTargets, err
 }
 
+func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) {
+	finput, err := os.Open(filePath)
+	if err != nil {
+		return 0, err
+	}
+	defer finput.Close()
+
+	err = format.Parse(finput, func(target string) bool {
+		target = strings.TrimSpace(target)
+		expandedTarget, countErr := r.countTargetFromRawTarget(target)
+		if countErr == nil && expandedTarget > 0 {
+			numTargets += expandedTarget
+			r.hm.Set(target, []byte("1")) //nolint
+		} else if r.options.SkipDedupe && errors.Is(countErr, duplicateTargetErr) {
+			if v, ok := r.hm.Get(target); ok {
+				cnt, _ := strconv.Atoi(string(v))
+				_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
+				numTargets += 1
+			}
+		}
+		return true
+	})
+	return numTargets, err
+}
+
 func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) {
 	if rawTarget == "" {
 		return 0, nil
@@ -2633,6 +2701,10 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 		}
 
 		clone.SetURL(resolvedURL)
+		// Update Host header to match resolved URL host (important after redirects)
+		if resolvedURL.Host != "" && resolvedURL.Host != clone.Host {
+			clone.Host = resolvedURL.Host
+		}
 		respFav, err := hp.Do(clone, httpx.UnsafeOptions{})
 		if err != nil || len(respFav.Data) == 0 {
 			tries++