admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat-add-md-output-support

Browse Source
This commit is contained in:
ayuxsec
2026-02-05 19:51:42 +05:30
committed by GitHub
parent 299cb60871 d54cb54817
commit c52b709976
28 changed files with 2680 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/PULL_REQUEST_TEMPLATE.md
+16
View File
@@ -0,0 +1,16 @@
## Proposed changes
<!-- Describe the overall picture of your modifications to help maintainers understand the pull request. PRs are required to be associated to their related issue tickets or feature request. -->
### Proof
<!-- How has this been tested? Please describe the tests that you ran to verify your changes. -->
## Checklist
<!-- Put an "x" in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code. -->
- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/httpx/tree/dev) branch
- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] I have added necessary documentation (if appropriate)
README.md
+28 -5
View File
@@ -92,9 +92,10 @@ Usage:
Flags:
INPUT:
-l, -list string input file containing list of hosts to process
-rr, -request string file containing raw request
-u, -target string[] input target host(s) to probe
-l, -list string input file containing list of hosts to process
-rr, -request string file containing raw request
-u, -target string[] input target host(s) to probe
-im, -input-mode string mode of input file (burp)
PROBES:
-sc, -status-code display response status-code
@@ -110,9 +111,11 @@ PROBES:
-title display page title
-bp, -body-preview display first N characters of response body (default 100)
-server, -web-server display server name
-td, -tech-detect display technology in use based on wappalyzer dataset
-td, -tech-detect display technology in use based on wappalyzer dataset
-cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection
-method display http request method
-cpe display CPE (Common Platform Enumeration) based on awesome-search-queries
-wp, -wordpress display WordPress plugins and themes
-method display http request method
-ws, -websocket display server using websocket
-ip display host ip
-cname display host cname
@@ -231,6 +234,7 @@ CONFIGURATIONS:
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-no-stdin Disable Stdin processing
-hae, -http-api-endpoint string experimental http api endpoint
-sf, -secret-file string path to secret file for authentication
DEBUG:
-health-check, -hc run diagnostic check up
@@ -277,9 +281,28 @@ For details about running httpx, see https://docs.projectdiscovery.io/tools/http
# Notes
- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
- Secret files can be used for domain-based authentication via `-sf secrets.yaml`. Supported auth types: `BasicAuth`, `BearerToken`, `Header`, `Cookie`, `Query`. Example:
```yaml
id: example-auth
info:
name: Example Auth Config
static:
- type: Header
domains:
- api.example.com
headers:
- key: X-API-Key
value: secret-key-here
- type: BasicAuth
domains-regex:
- ".*\\.internal\\.com$"
username: admin
password: secret
```
- The following flags should be used for specific use cases instead of running them as default with other probes:
- `-ports`
- `-path`
common/authprovider/authx/basic_auth.go
+31
View File
@@ -0,0 +1,31 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
var (
_ AuthStrategy = &BasicAuthStrategy{}
)
// BasicAuthStrategy is a strategy for basic auth
type BasicAuthStrategy struct {
Data *Secret
}
// NewBasicAuthStrategy creates a new basic auth strategy
func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy {
return &BasicAuthStrategy{Data: data}
}
// Apply applies the basic auth strategy to the request
func (s *BasicAuthStrategy) Apply(req *http.Request) {
req.SetBasicAuth(s.Data.Username, s.Data.Password)
}
// ApplyOnRR applies the basic auth strategy to the retryable request
func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
req.SetBasicAuth(s.Data.Username, s.Data.Password)
}
common/authprovider/authx/bearer_auth.go
+31
View File
@@ -0,0 +1,31 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
var (
_ AuthStrategy = &BearerTokenAuthStrategy{}
)
// BearerTokenAuthStrategy is a strategy for bearer token auth
type BearerTokenAuthStrategy struct {
Data *Secret
}
// NewBearerTokenAuthStrategy creates a new bearer token auth strategy
func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy {
return &BearerTokenAuthStrategy{Data: data}
}
// Apply applies the bearer token auth strategy to the request
func (s *BearerTokenAuthStrategy) Apply(req *http.Request) {
req.Header.Set("Authorization", "Bearer "+s.Data.Token)
}
// ApplyOnRR applies the bearer token auth strategy to the retryable request
func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
req.Header.Set("Authorization", "Bearer "+s.Data.Token)
}
common/authprovider/authx/cookies_auth.go
+62
View File
@@ -0,0 +1,62 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
var (
_ AuthStrategy = &CookiesAuthStrategy{}
)
// CookiesAuthStrategy is a strategy for cookies auth
type CookiesAuthStrategy struct {
Data *Secret
}
// NewCookiesAuthStrategy creates a new cookies auth strategy
func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy {
return &CookiesAuthStrategy{Data: data}
}
// Apply applies the cookies auth strategy to the request
func (s *CookiesAuthStrategy) Apply(req *http.Request) {
for _, cookie := range s.Data.Cookies {
req.AddCookie(&http.Cookie{
Name: cookie.Key,
Value: cookie.Value,
})
}
}
// ApplyOnRR applies the cookies auth strategy to the retryable request
func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
// Build a set of cookie names to replace
newCookieNames := make(map[string]struct{}, len(s.Data.Cookies))
for _, cookie := range s.Data.Cookies {
newCookieNames[cookie.Key] = struct{}{}
}
// Filter existing cookies, keeping only those not being replaced
existingCookies := req.Cookies()
filteredCookies := make([]*http.Cookie, 0, len(existingCookies))
for _, cookie := range existingCookies {
if _, shouldReplace := newCookieNames[cookie.Name]; !shouldReplace {
filteredCookies = append(filteredCookies, cookie)
}
}
// Clear and reset cookies
req.Header.Del("Cookie")
for _, cookie := range filteredCookies {
req.AddCookie(cookie)
}
// Add new cookies
for _, cookie := range s.Data.Cookies {
req.AddCookie(&http.Cookie{
Name: cookie.Key,
Value: cookie.Value,
})
}
}
common/authprovider/authx/file.go
+244
View File
@@ -0,0 +1,244 @@
package authx
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/projectdiscovery/utils/errkit"
"github.com/projectdiscovery/utils/generic"
stringsutil "github.com/projectdiscovery/utils/strings"
"gopkg.in/yaml.v3"
)
type AuthType string
const (
BasicAuth AuthType = "BasicAuth"
BearerTokenAuth AuthType = "BearerToken"
HeadersAuth AuthType = "Header"
CookiesAuth AuthType = "Cookie"
QueryAuth AuthType = "Query"
)
// SupportedAuthTypes returns the supported auth types
func SupportedAuthTypes() []string {
return []string{
string(BasicAuth),
string(BearerTokenAuth),
string(HeadersAuth),
string(CookiesAuth),
string(QueryAuth),
}
}
// Authx is a struct for secrets or credentials file
type Authx struct {
ID string `json:"id" yaml:"id"`
Info AuthFileInfo `json:"info" yaml:"info"`
Secrets []Secret `json:"static" yaml:"static"`
}
type AuthFileInfo struct {
Name string `json:"name" yaml:"name"`
Author string `json:"author" yaml:"author"`
Severity string `json:"severity" yaml:"severity"`
Description string `json:"description" yaml:"description"`
}
// Secret is a struct for secret or credential
type Secret struct {
Type string `json:"type" yaml:"type"`
Domains []string `json:"domains" yaml:"domains"`
DomainsRegex []string `json:"domains-regex" yaml:"domains-regex"`
Headers []KV `json:"headers" yaml:"headers"` // Headers preserve exact casing (useful for case-sensitive APIs)
Cookies []Cookie `json:"cookies" yaml:"cookies"`
Params []KV `json:"params" yaml:"params"`
Username string `json:"username" yaml:"username"` // can be either email or username
Password string `json:"password" yaml:"password"`
Token string `json:"token" yaml:"token"` // Bearer Auth token
}
// GetStrategy returns the auth strategy for the secret
func (s *Secret) GetStrategy() AuthStrategy {
switch {
case strings.EqualFold(s.Type, string(BasicAuth)):
return NewBasicAuthStrategy(s)
case strings.EqualFold(s.Type, string(BearerTokenAuth)):
return NewBearerTokenAuthStrategy(s)
case strings.EqualFold(s.Type, string(HeadersAuth)):
return NewHeadersAuthStrategy(s)
case strings.EqualFold(s.Type, string(CookiesAuth)):
return NewCookiesAuthStrategy(s)
case strings.EqualFold(s.Type, string(QueryAuth)):
return NewQueryAuthStrategy(s)
}
return nil
}
func (s *Secret) Validate() error {
if !stringsutil.EqualFoldAny(s.Type, SupportedAuthTypes()...) {
return fmt.Errorf("invalid type: %s", s.Type)
}
if len(s.Domains) == 0 && len(s.DomainsRegex) == 0 {
return fmt.Errorf("domains or domains-regex cannot be empty")
}
if len(s.DomainsRegex) > 0 {
for _, domain := range s.DomainsRegex {
_, err := regexp.Compile(domain)
if err != nil {
return fmt.Errorf("invalid domain regex: %s", domain)
}
}
}
switch {
case strings.EqualFold(s.Type, string(BasicAuth)):
if s.Username == "" {
return fmt.Errorf("username cannot be empty in basic auth")
}
if s.Password == "" {
return fmt.Errorf("password cannot be empty in basic auth")
}
case strings.EqualFold(s.Type, string(BearerTokenAuth)):
if s.Token == "" {
return fmt.Errorf("token cannot be empty in bearer token auth")
}
case strings.EqualFold(s.Type, string(HeadersAuth)):
if len(s.Headers) == 0 {
return fmt.Errorf("headers cannot be empty in headers auth")
}
for _, header := range s.Headers {
if err := header.Validate(); err != nil {
return fmt.Errorf("invalid header in headersAuth: %s", err)
}
}
case strings.EqualFold(s.Type, string(CookiesAuth)):
if len(s.Cookies) == 0 {
return fmt.Errorf("cookies cannot be empty in cookies auth")
}
for _, cookie := range s.Cookies {
if cookie.Raw != "" {
if err := cookie.Parse(); err != nil {
return fmt.Errorf("invalid raw cookie in cookiesAuth: %s", err)
}
}
if err := cookie.Validate(); err != nil {
return fmt.Errorf("invalid cookie in cookiesAuth: %s", err)
}
}
case strings.EqualFold(s.Type, string(QueryAuth)):
if len(s.Params) == 0 {
return fmt.Errorf("query cannot be empty in query auth")
}
for _, query := range s.Params {
if err := query.Validate(); err != nil {
return fmt.Errorf("invalid query in queryAuth: %s", err)
}
}
}
return nil
}
type KV struct {
Key string `json:"key" yaml:"key"` // Header key (preserves exact casing)
Value string `json:"value" yaml:"value"`
}
func (k *KV) Validate() error {
if k.Key == "" {
return fmt.Errorf("key cannot be empty")
}
if k.Value == "" {
return fmt.Errorf("value cannot be empty")
}
return nil
}
type Cookie struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
Raw string `json:"raw" yaml:"raw"`
}
func (c *Cookie) Validate() error {
if c.Raw != "" {
return nil
}
if c.Key == "" {
return fmt.Errorf("key cannot be empty")
}
if c.Value == "" {
return fmt.Errorf("value cannot be empty")
}
return nil
}
// Parse parses the cookie
// in raw the cookie is in format of
// Set-Cookie: <cookie-name>=<cookie-value>; Expires=<date>; Path=<path>; Domain=<domain_name>; Secure; HttpOnly
func (c *Cookie) Parse() error {
if c.Raw == "" {
return fmt.Errorf("raw cookie cannot be empty")
}
tmp := strings.TrimPrefix(c.Raw, "Set-Cookie: ")
slice := strings.Split(tmp, ";")
if len(slice) == 0 {
return fmt.Errorf("invalid raw cookie no ; found")
}
// first element is the cookie name and value
// Use SplitN to preserve '=' characters in the cookie value
cookie := strings.SplitN(slice[0], "=", 2)
if len(cookie) != 2 {
return fmt.Errorf("invalid raw cookie: missing '=' in cookie name=value: %s", c.Raw)
}
c.Key = strings.TrimSpace(cookie[0])
c.Value = strings.TrimSpace(cookie[1])
if c.Key == "" {
return fmt.Errorf("invalid raw cookie: empty cookie name: %s", c.Raw)
}
return nil
}
// GetAuthDataFromFile reads the auth data from file
func GetAuthDataFromFile(file string) (*Authx, error) {
ext := strings.ToLower(filepath.Ext(file))
if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") {
return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext)
}
bin, err := os.ReadFile(file)
if err != nil {
return nil, err
}
if ext == ".yml" || ext == ".yaml" {
return GetAuthDataFromYAML(bin)
}
return GetAuthDataFromJSON(bin)
}
// GetAuthDataFromYAML reads the auth data from yaml
func GetAuthDataFromYAML(data []byte) (*Authx, error) {
var auth Authx
err := yaml.Unmarshal(data, &auth)
if err != nil {
errorErr := errkit.FromError(err)
errorErr.Msgf("could not unmarshal yaml")
return nil, errorErr
}
return &auth, nil
}
// GetAuthDataFromJSON reads the auth data from json
func GetAuthDataFromJSON(data []byte) (*Authx, error) {
var auth Authx
err := json.Unmarshal(data, &auth)
if err != nil {
errorErr := errkit.FromError(err)
errorErr.Msgf("could not unmarshal json")
return nil, errorErr
}
return &auth, nil
}
common/authprovider/authx/file_test.go
+408
View File
@@ -0,0 +1,408 @@
package authx
import (
"os"
"path/filepath"
"testing"
)
func TestCookieParse(t *testing.T) {
tests := []struct {
name string
raw string
wantKey string
wantValue string
wantErr bool
}{
{
name: "simple cookie",
raw: "session=abc123",
wantKey: "session",
wantValue: "abc123",
wantErr: false,
},
{
name: "cookie with Set-Cookie prefix",
raw: "Set-Cookie: session=abc123; Path=/",
wantKey: "session",
wantValue: "abc123",
wantErr: false,
},
{
name: "cookie with equals in value",
raw: "token=eyJhbGciOiJIUzI1NiJ9==; Path=/",
wantKey: "token",
wantValue: "eyJhbGciOiJIUzI1NiJ9==",
wantErr: false,
},
{
name: "cookie with spaces",
raw: " session = abc123 ; Path=/",
wantKey: "session",
wantValue: "abc123",
wantErr: false,
},
{
name: "empty raw",
raw: "",
wantErr: true,
},
{
name: "missing equals",
raw: "sessionabc123; Path=/",
wantErr: true,
},
{
name: "empty key",
raw: "=abc123; Path=/",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Cookie{Raw: tt.raw}
err := c.Parse()
if (err != nil) != tt.wantErr {
t.Errorf("Cookie.Parse() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr {
if c.Key != tt.wantKey {
t.Errorf("Cookie.Parse() Key = %v, want %v", c.Key, tt.wantKey)
}
if c.Value != tt.wantValue {
t.Errorf("Cookie.Parse() Value = %v, want %v", c.Value, tt.wantValue)
}
}
})
}
}
func TestSecretValidate(t *testing.T) {
tests := []struct {
name string
secret Secret
wantErr bool
}{
{
name: "valid basic auth",
secret: Secret{
Type: "BasicAuth",
Domains: []string{"example.com"},
Username: "user",
Password: "pass",
},
wantErr: false,
},
{
name: "valid bearer token",
secret: Secret{
Type: "BearerToken",
Domains: []string{"example.com"},
Token: "abc123",
},
wantErr: false,
},
{
name: "valid header auth",
secret: Secret{
Type: "Header",
Domains: []string{"example.com"},
Headers: []KV{{Key: "X-API-Key", Value: "secret"}},
},
wantErr: false,
},
{
name: "valid cookie auth",
secret: Secret{
Type: "Cookie",
Domains: []string{"example.com"},
Cookies: []Cookie{{Key: "session", Value: "abc123"}},
},
wantErr: false,
},
{
name: "valid query auth",
secret: Secret{
Type: "Query",
Domains: []string{"example.com"},
Params: []KV{{Key: "api_key", Value: "secret"}},
},
wantErr: false,
},
{
name: "invalid type",
secret: Secret{
Type: "InvalidType",
Domains: []string{"example.com"},
},
wantErr: true,
},
{
name: "missing domains",
secret: Secret{
Type: "BasicAuth",
Username: "user",
Password: "pass",
},
wantErr: true,
},
{
name: "basic auth missing username",
secret: Secret{
Type: "BasicAuth",
Domains: []string{"example.com"},
Password: "pass",
},
wantErr: true,
},
{
name: "basic auth missing password",
secret: Secret{
Type: "BasicAuth",
Domains: []string{"example.com"},
Username: "user",
},
wantErr: true,
},
{
name: "bearer auth missing token",
secret: Secret{
Type: "BearerToken",
Domains: []string{"example.com"},
},
wantErr: true,
},
{
name: "header auth missing headers",
secret: Secret{
Type: "Header",
Domains: []string{"example.com"},
},
wantErr: true,
},
{
name: "cookie auth missing cookies",
secret: Secret{
Type: "Cookie",
Domains: []string{"example.com"},
},
wantErr: true,
},
{
name: "query auth missing params",
secret: Secret{
Type: "Query",
Domains: []string{"example.com"},
},
wantErr: true,
},
{
name: "valid domain regex",
secret: Secret{
Type: "BearerToken",
DomainsRegex: []string{".*\\.example\\.com"},
Token: "abc123",
},
wantErr: false,
},
{
name: "invalid domain regex",
secret: Secret{
Type: "BearerToken",
DomainsRegex: []string{"[invalid"},
Token: "abc123",
},
wantErr: true,
},
{
name: "case insensitive type",
secret: Secret{
Type: "basicauth",
Domains: []string{"example.com"},
Username: "user",
Password: "pass",
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.secret.Validate()
if (err != nil) != tt.wantErr {
t.Errorf("Secret.Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestGetAuthDataFromFile(t *testing.T) {
// Create temp directory for test files
tmpDir := t.TempDir()
yamlContent := `id: test
info:
name: test
static:
- type: BasicAuth
domains:
- example.com
username: user
password: pass
`
jsonContent := `{
"id": "test",
"info": {"name": "test"},
"static": [
{
"type": "BasicAuth",
"domains": ["example.com"],
"username": "user",
"password": "pass"
}
]
}`
tests := []struct {
name string
filename string
content string
wantErr bool
}{
{
name: "yaml file lowercase",
filename: "secrets.yaml",
content: yamlContent,
wantErr: false,
},
{
name: "yml file lowercase",
filename: "secrets.yml",
content: yamlContent,
wantErr: false,
},
{
name: "json file lowercase",
filename: "secrets.json",
content: jsonContent,
wantErr: false,
},
{
name: "yaml file uppercase",
filename: "secrets.YAML",
content: yamlContent,
wantErr: false,
},
{
name: "json file uppercase",
filename: "secrets.JSON",
content: jsonContent,
wantErr: false,
},
{
name: "invalid extension",
filename: "secrets.txt",
content: yamlContent,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create test file
filePath := filepath.Join(tmpDir, tt.filename)
err := os.WriteFile(filePath, []byte(tt.content), 0644)
if err != nil {
t.Fatalf("Failed to create test file: %v", err)
}
_, err = GetAuthDataFromFile(filePath)
if (err != nil) != tt.wantErr {
t.Errorf("GetAuthDataFromFile() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestSecretGetStrategy(t *testing.T) {
tests := []struct {
name string
secret Secret
wantType string
wantNil bool
}{
{
name: "basic auth strategy",
secret: Secret{
Type: "BasicAuth",
Username: "user",
Password: "pass",
},
wantType: "*authx.BasicAuthStrategy",
wantNil: false,
},
{
name: "bearer token strategy",
secret: Secret{
Type: "BearerToken",
Token: "abc123",
},
wantType: "*authx.BearerTokenAuthStrategy",
wantNil: false,
},
{
name: "header strategy",
secret: Secret{
Type: "Header",
Headers: []KV{{Key: "X-API-Key", Value: "secret"}},
},
wantType: "*authx.HeadersAuthStrategy",
wantNil: false,
},
{
name: "cookie strategy",
secret: Secret{
Type: "Cookie",
Cookies: []Cookie{{Key: "session", Value: "abc123"}},
},
wantType: "*authx.CookiesAuthStrategy",
wantNil: false,
},
{
name: "query strategy",
secret: Secret{
Type: "Query",
Params: []KV{{Key: "api_key", Value: "secret"}},
},
wantType: "*authx.QueryAuthStrategy",
wantNil: false,
},
{
name: "unknown type returns nil",
secret: Secret{
Type: "UnknownType",
},
wantNil: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
strategy := tt.secret.GetStrategy()
if tt.wantNil {
if strategy != nil {
t.Errorf("GetStrategy() = %T, want nil", strategy)
}
} else {
if strategy == nil {
t.Errorf("GetStrategy() = nil, want %s", tt.wantType)
}
}
})
}
}
common/authprovider/authx/headers_auth.go
+39
View File
@@ -0,0 +1,39 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
var (
_ AuthStrategy = &HeadersAuthStrategy{}
)
// HeadersAuthStrategy is a strategy for headers auth
type HeadersAuthStrategy struct {
Data *Secret
}
// NewHeadersAuthStrategy creates a new headers auth strategy
func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy {
return &HeadersAuthStrategy{Data: data}
}
// Apply applies the headers auth strategy to the request
// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
// This is useful for APIs that require case-sensitive header names
func (s *HeadersAuthStrategy) Apply(req *http.Request) {
for _, header := range s.Data.Headers {
req.Header[header.Key] = []string{header.Value}
}
}
// ApplyOnRR applies the headers auth strategy to the retryable request
// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken)
// This is useful for APIs that require case-sensitive header names
func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
for _, header := range s.Data.Headers {
req.Header[header.Key] = []string{header.Value}
}
}
common/authprovider/authx/query_auth.go
+42
View File
@@ -0,0 +1,42 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
)
var (
_ AuthStrategy = &QueryAuthStrategy{}
)
// QueryAuthStrategy is a strategy for query auth
type QueryAuthStrategy struct {
Data *Secret
}
// NewQueryAuthStrategy creates a new query auth strategy
func NewQueryAuthStrategy(data *Secret) *QueryAuthStrategy {
return &QueryAuthStrategy{Data: data}
}
// Apply applies the query auth strategy to the request
func (s *QueryAuthStrategy) Apply(req *http.Request) {
q := urlutil.NewOrderedParams()
q.Decode(req.URL.RawQuery)
for _, p := range s.Data.Params {
q.Add(p.Key, p.Value)
}
req.URL.RawQuery = q.Encode()
}
// ApplyOnRR applies the query auth strategy to the retryable request
func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) {
q := urlutil.NewOrderedParams()
q.Decode(req.Request.URL.RawQuery)
for _, p := range s.Data.Params {
q.Add(p.Key, p.Value)
}
req.Request.URL.RawQuery = q.Encode()
}
common/authprovider/authx/strategy.go
+16
View File
@@ -0,0 +1,16 @@
package authx
import (
"net/http"
"github.com/projectdiscovery/retryablehttp-go"
)
// AuthStrategy is an interface for auth strategies
// basic auth , bearer token, headers, cookies, query
type AuthStrategy interface {
// Apply applies the strategy to the request
Apply(*http.Request)
// ApplyOnRR applies the strategy to the retryable request
ApplyOnRR(*retryablehttp.Request)
}
common/authprovider/authx/strategy_test.go
+217
View File
@@ -0,0 +1,217 @@
package authx
import (
"net/http"
"testing"
"github.com/projectdiscovery/retryablehttp-go"
)
func TestBasicAuthStrategy(t *testing.T) {
secret := &Secret{
Username: "user",
Password: "pass",
}
strategy := NewBasicAuthStrategy(secret)
t.Run("Apply", func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com", nil)
strategy.Apply(req)
user, pass, ok := req.BasicAuth()
if !ok {
t.Error("Basic auth not set")
}
if user != "user" {
t.Errorf("Username = %v, want user", user)
}
if pass != "pass" {
t.Errorf("Password = %v, want pass", pass)
}
})
t.Run("ApplyOnRR", func(t *testing.T) {
req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
strategy.ApplyOnRR(req)
user, pass, ok := req.BasicAuth()
if !ok {
t.Error("Basic auth not set")
}
if user != "user" {
t.Errorf("Username = %v, want user", user)
}
if pass != "pass" {
t.Errorf("Password = %v, want pass", pass)
}
})
}
func TestBearerTokenAuthStrategy(t *testing.T) {
secret := &Secret{
Token: "mytoken123",
}
strategy := NewBearerTokenAuthStrategy(secret)
t.Run("Apply", func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com", nil)
strategy.Apply(req)
auth := req.Header.Get("Authorization")
expected := "Bearer mytoken123"
if auth != expected {
t.Errorf("Authorization = %v, want %v", auth, expected)
}
})
t.Run("ApplyOnRR", func(t *testing.T) {
req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
strategy.ApplyOnRR(req)
auth := req.Header.Get("Authorization")
expected := "Bearer mytoken123"
if auth != expected {
t.Errorf("Authorization = %v, want %v", auth, expected)
}
})
}
func TestHeadersAuthStrategy(t *testing.T) {
// Headers strategy preserves exact casing, so use exact key names
secret := &Secret{
Headers: []KV{
{Key: "X-API-Key", Value: "secret123"},
{Key: "X-Custom", Value: "value"},
},
}
strategy := NewHeadersAuthStrategy(secret)
t.Run("Apply", func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com", nil)
strategy.Apply(req)
// Use direct map access since headers preserve exact casing
//nolint
if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" {
t.Errorf("X-API-Key = %v, want [secret123]", got)
}
if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" {
t.Errorf("X-Custom = %v, want [value]", got)
}
})
t.Run("ApplyOnRR", func(t *testing.T) {
req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
strategy.ApplyOnRR(req)
// Use direct map access since headers preserve exact casing
//nolint
if got := req.Header["X-API-Key"]; len(got) == 0 || got[0] != "secret123" {
t.Errorf("X-API-Key = %v, want [secret123]", got)
}
if got := req.Header["X-Custom"]; len(got) == 0 || got[0] != "value" {
t.Errorf("X-Custom = %v, want [value]", got)
}
})
}
func TestCookiesAuthStrategy(t *testing.T) {
secret := &Secret{
Cookies: []Cookie{
{Key: "session", Value: "abc123"},
{Key: "auth", Value: "xyz789"},
},
}
strategy := NewCookiesAuthStrategy(secret)
t.Run("Apply", func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com", nil)
strategy.Apply(req)
cookies := req.Cookies()
if len(cookies) != 2 {
t.Errorf("Expected 2 cookies, got %d", len(cookies))
}
found := make(map[string]string)
for _, c := range cookies {
found[c.Name] = c.Value
}
if found["session"] != "abc123" {
t.Errorf("session cookie = %v, want abc123", found["session"])
}
if found["auth"] != "xyz789" {
t.Errorf("auth cookie = %v, want xyz789", found["auth"])
}
})
t.Run("ApplyOnRR replaces existing cookies", func(t *testing.T) {
req, _ := retryablehttp.NewRequest("GET", "http://example.com", nil)
// Add existing cookie that should be replaced
req.AddCookie(&http.Cookie{Name: "session", Value: "old_value"})
// Add existing cookie that should be kept
req.AddCookie(&http.Cookie{Name: "other", Value: "keep_me"})
strategy.ApplyOnRR(req)
cookies := req.Cookies()
found := make(map[string]string)
for _, c := range cookies {
found[c.Name] = c.Value
}
// New cookie values should override old ones
if found["session"] != "abc123" {
t.Errorf("session cookie = %v, want abc123", found["session"])
}
if found["auth"] != "xyz789" {
t.Errorf("auth cookie = %v, want xyz789", found["auth"])
}
// Existing non-replaced cookie should be preserved
if found["other"] != "keep_me" {
t.Errorf("other cookie = %v, want keep_me", found["other"])
}
})
}
func TestQueryAuthStrategy(t *testing.T) {
secret := &Secret{
Params: []KV{
{Key: "api_key", Value: "secret123"},
{Key: "token", Value: "abc"},
},
}
strategy := NewQueryAuthStrategy(secret)
t.Run("Apply", func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com/path?existing=value", nil)
strategy.Apply(req)
query := req.URL.Query()
if got := query.Get("api_key"); got != "secret123" {
t.Errorf("api_key = %v, want secret123", got)
}
if got := query.Get("token"); got != "abc" {
t.Errorf("token = %v, want abc", got)
}
if got := query.Get("existing"); got != "value" {
t.Errorf("existing = %v, want value", got)
}
})
t.Run("ApplyOnRR", func(t *testing.T) {
req, _ := retryablehttp.NewRequest("GET", "http://example.com/path?existing=value", nil)
strategy.ApplyOnRR(req)
query := req.Request.URL.Query()
if got := query.Get("api_key"); got != "secret123" {
t.Errorf("api_key = %v, want secret123", got)
}
if got := query.Get("token"); got != "abc" {
t.Errorf("token = %v, want abc", got)
}
if got := query.Get("existing"); got != "value" {
t.Errorf("existing = %v, want value", got)
}
})
}
common/authprovider/file.go
+114
View File
@@ -0,0 +1,114 @@
package authprovider
import (
"net"
"net/url"
"regexp"
"strings"
"github.com/projectdiscovery/httpx/common/authprovider/authx"
"github.com/projectdiscovery/utils/errkit"
urlutil "github.com/projectdiscovery/utils/url"
)
// FileAuthProvider is an auth provider for file based auth
// it accepts a secrets file and returns its provider
type FileAuthProvider struct {
Path string
store *authx.Authx
compiled map[*regexp.Regexp][]authx.AuthStrategy
domains map[string][]authx.AuthStrategy
}
// NewFileAuthProvider creates a new file based auth provider
func NewFileAuthProvider(path string) (AuthProvider, error) {
store, err := authx.GetAuthDataFromFile(path)
if err != nil {
return nil, err
}
if len(store.Secrets) == 0 {
return nil, ErrNoSecrets
}
for _, secret := range store.Secrets {
if err := secret.Validate(); err != nil {
errorErr := errkit.FromError(err)
errorErr.Msgf("invalid secret in file: %s", path)
return nil, errorErr
}
}
f := &FileAuthProvider{Path: path, store: store}
f.init()
return f, nil
}
// init initializes the file auth provider
func (f *FileAuthProvider) init() {
for _, _secret := range f.store.Secrets {
secret := _secret // capture loop variable for use in GetStrategy()
if len(secret.DomainsRegex) > 0 {
for _, domain := range secret.DomainsRegex {
if f.compiled == nil {
f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy)
}
compiled, err := regexp.Compile(domain)
if err != nil {
continue
}
if ss, ok := f.compiled[compiled]; ok {
f.compiled[compiled] = append(ss, secret.GetStrategy())
} else {
f.compiled[compiled] = []authx.AuthStrategy{secret.GetStrategy()}
}
}
}
for _, domain := range secret.Domains {
if f.domains == nil {
f.domains = make(map[string][]authx.AuthStrategy)
}
domain = strings.TrimSpace(domain)
domain = strings.TrimSuffix(domain, ":80")
domain = strings.TrimSuffix(domain, ":443")
if ss, ok := f.domains[domain]; ok {
f.domains[domain] = append(ss, secret.GetStrategy())
} else {
f.domains[domain] = []authx.AuthStrategy{secret.GetStrategy()}
}
}
}
}
// LookupAddr looks up a given domain/address and returns appropriate auth strategy
func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy {
var strategies []authx.AuthStrategy
if strings.Contains(addr, ":") {
// strip default ports (80/443) for consistent domain matching
host, port, err := net.SplitHostPort(addr)
if err == nil && (port == "80" || port == "443") {
addr = host
}
}
for domain, strategy := range f.domains {
if strings.EqualFold(domain, addr) {
strategies = append(strategies, strategy...)
}
}
for compiled, strategy := range f.compiled {
if compiled.MatchString(addr) {
strategies = append(strategies, strategy...)
}
}
return strategies
}
// LookupURL looks up a given URL and returns appropriate auth strategy
func (f *FileAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy {
return f.LookupAddr(u.Host)
}
// LookupURLX looks up a given URL and returns appropriate auth strategy
func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy {
return f.LookupAddr(u.Host)
}
common/authprovider/interface.go
+53
View File
@@ -0,0 +1,53 @@
// TODO: This package should be abstracted out to projectdiscovery/utils
// so it can be shared between httpx, nuclei, and other tools.
package authprovider
import (
"fmt"
"net/url"
"github.com/projectdiscovery/httpx/common/authprovider/authx"
urlutil "github.com/projectdiscovery/utils/url"
)
var (
ErrNoSecrets = fmt.Errorf("no secrets in given provider")
)
var (
_ AuthProvider = &FileAuthProvider{}
)
// AuthProvider is an interface for auth providers
// It implements a data structure suitable for quick lookup and retrieval
// of auth strategies
type AuthProvider interface {
// LookupAddr looks up a given domain/address and returns appropriate auth strategy
// for it (accepted inputs are scanme.sh or scanme.sh:443)
LookupAddr(string) []authx.AuthStrategy
// LookupURL looks up a given URL and returns appropriate auth strategy
// it accepts a valid url struct and returns the auth strategy
LookupURL(*url.URL) []authx.AuthStrategy
// LookupURLX looks up a given URL and returns appropriate auth strategy
// it accepts pd url struct (i.e urlutil.URL) and returns the auth strategy
LookupURLX(*urlutil.URL) []authx.AuthStrategy
}
// AuthProviderOptions contains options for the auth provider
type AuthProviderOptions struct {
// File based auth provider options
SecretsFiles []string
}
// NewAuthProvider creates a new auth provider from the given options
func NewAuthProvider(options *AuthProviderOptions) (AuthProvider, error) {
var providers []AuthProvider
for _, file := range options.SecretsFiles {
provider, err := NewFileAuthProvider(file)
if err != nil {
return nil, err
}
providers = append(providers, provider)
}
return NewMultiAuthProvider(providers...), nil
}
common/authprovider/multi.go
+50
View File
@@ -0,0 +1,50 @@
package authprovider
import (
"net/url"
"github.com/projectdiscovery/httpx/common/authprovider/authx"
urlutil "github.com/projectdiscovery/utils/url"
)
// MultiAuthProvider is a convenience wrapper for multiple auth providers
// it returns the first matching auth strategy for a given domain
// if there are multiple auth strategies for a given domain, it returns the first one
type MultiAuthProvider struct {
Providers []AuthProvider
}
// NewMultiAuthProvider creates a new multi auth provider
func NewMultiAuthProvider(providers ...AuthProvider) AuthProvider {
return &MultiAuthProvider{Providers: providers}
}
func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupAddr(host)
if len(strategy) > 0 {
return strategy
}
}
return nil
}
func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupURL(u)
if len(strategy) > 0 {
return strategy
}
}
return nil
}
func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy {
for _, provider := range m.Providers {
strategy := provider.LookupURLX(u)
if len(strategy) > 0 {
return strategy
}
}
return nil
}
common/authprovider/provider_test.go
+256
View File
@@ -0,0 +1,256 @@
package authprovider
import (
"net/url"
"os"
"path/filepath"
"testing"
urlutil "github.com/projectdiscovery/utils/url"
)
func createTestSecretsFile(t *testing.T, content string) string {
t.Helper()
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "secrets.yaml")
err := os.WriteFile(filePath, []byte(content), 0644)
if err != nil {
t.Fatalf("Failed to create test secrets file: %v", err)
}
return filePath
}
func TestFileAuthProviderLookupAddr(t *testing.T) {
content := `id: test
info:
name: test
static:
- type: BasicAuth
domains:
- example.com
- api.example.com:443
username: user
password: pass
- type: BearerToken
domains-regex:
- ".*\\.test\\.com"
token: regextoken
`
filePath := createTestSecretsFile(t, content)
provider, err := NewFileAuthProvider(filePath)
if err != nil {
t.Fatalf("NewFileAuthProvider() error = %v", err)
}
tests := []struct {
name string
addr string
wantCount int
}{
{
name: "exact match",
addr: "example.com",
wantCount: 1,
},
{
name: "exact match case insensitive",
addr: "EXAMPLE.COM",
wantCount: 1,
},
{
name: "with port 443 normalized",
addr: "example.com:443",
wantCount: 1,
},
{
name: "with port 80 normalized",
addr: "example.com:80",
wantCount: 1,
},
{
name: "subdomain exact match",
addr: "api.example.com",
wantCount: 1,
},
{
name: "regex match",
addr: "foo.test.com",
wantCount: 1,
},
{
name: "regex match subdomain",
addr: "bar.baz.test.com",
wantCount: 1,
},
{
name: "no match",
addr: "unknown.com",
wantCount: 0,
},
{
name: "non-standard port not normalized",
addr: "example.com:8080",
wantCount: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
strategies := provider.LookupAddr(tt.addr)
if len(strategies) != tt.wantCount {
t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount)
}
})
}
}
func TestFileAuthProviderLookupURL(t *testing.T) {
content := `id: test
info:
name: test
static:
- type: BasicAuth
domains:
- example.com
username: user
password: pass
`
filePath := createTestSecretsFile(t, content)
provider, err := NewFileAuthProvider(filePath)
if err != nil {
t.Fatalf("NewFileAuthProvider() error = %v", err)
}
t.Run("LookupURL", func(t *testing.T) {
u, _ := url.Parse("https://example.com/path")
strategies := provider.LookupURL(u)
if len(strategies) != 1 {
t.Errorf("LookupURL() returned %d strategies, want 1", len(strategies))
}
})
t.Run("LookupURLX", func(t *testing.T) {
u, _ := urlutil.Parse("https://example.com/path")
strategies := provider.LookupURLX(u)
if len(strategies) != 1 {
t.Errorf("LookupURLX() returned %d strategies, want 1", len(strategies))
}
})
}
func TestMultiAuthProvider(t *testing.T) {
content1 := `id: test1
info:
name: test1
static:
- type: BasicAuth
domains:
- first.com
username: user1
password: pass1
`
content2 := `id: test2
info:
name: test2
static:
- type: BearerToken
domains:
- second.com
token: token2
`
filePath1 := createTestSecretsFile(t, content1)
provider1, err := NewFileAuthProvider(filePath1)
if err != nil {
t.Fatalf("NewFileAuthProvider() error = %v", err)
}
// Create second file in different temp dir
tmpDir2 := t.TempDir()
filePath2 := filepath.Join(tmpDir2, "secrets2.yaml")
err = os.WriteFile(filePath2, []byte(content2), 0644)
if err != nil {
t.Fatalf("Failed to create test secrets file: %v", err)
}
provider2, err := NewFileAuthProvider(filePath2)
if err != nil {
t.Fatalf("NewFileAuthProvider() error = %v", err)
}
multi := NewMultiAuthProvider(provider1, provider2)
tests := []struct {
name string
addr string
wantCount int
}{
{
name: "match first provider",
addr: "first.com",
wantCount: 1,
},
{
name: "match second provider",
addr: "second.com",
wantCount: 1,
},
{
name: "no match",
addr: "third.com",
wantCount: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
strategies := multi.LookupAddr(tt.addr)
if len(strategies) != tt.wantCount {
t.Errorf("LookupAddr(%q) returned %d strategies, want %d", tt.addr, len(strategies), tt.wantCount)
}
})
}
}
func TestNewFileAuthProviderErrors(t *testing.T) {
tests := []struct {
name string
content string
wantErr bool
}{
{
name: "empty secrets",
content: `id: test`,
wantErr: true,
},
{
name: "invalid secret type",
content: `id: test
static:
- type: InvalidType
domains:
- example.com
`,
wantErr: true,
},
{
name: "missing required field",
content: `id: test
static:
- type: BasicAuth
domains:
- example.com
username: user
`,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filePath := createTestSecretsFile(t, tt.content)
_, err := NewFileAuthProvider(filePath)
if (err != nil) != tt.wantErr {
t.Errorf("NewFileAuthProvider() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
common/inputformats/burp.go
+44
View File
@@ -0,0 +1,44 @@
package inputformats
import (
"io"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/seh-msft/burpxml"
)
// BurpFormat is a Burp Suite XML file parser
type BurpFormat struct{}
// NewBurpFormat creates a new Burp XML file parser
func NewBurpFormat() *BurpFormat {
return &BurpFormat{}
}
var _ Format = &BurpFormat{}
// Name returns the name of the format
func (b *BurpFormat) Name() string {
return "burp"
}
// Parse parses the Burp XML input and calls the provided callback
// function for each URL it discovers.
func (b *BurpFormat) Parse(input io.Reader, callback func(url string) bool) error {
items, err := burpxml.Parse(input, true)
if err != nil {
return errors.Wrap(err, "could not parse burp xml")
}
for i, item := range items.Items {
if item.Url == "" {
gologger.Debug().Msgf("Skipping burp item %d: empty URL", i)
continue
}
if !callback(item.Url) {
break
}
}
return nil
}
common/inputformats/burp_test.go
+169
View File
@@ -0,0 +1,169 @@
package inputformats
import (
"strings"
"testing"
)
func TestBurpFormat_Name(t *testing.T) {
b := NewBurpFormat()
if b.Name() != "burp" {
t.Errorf("Expected name 'burp', got '%s'", b.Name())
}
}
func TestBurpFormat_Parse(t *testing.T) {
burpXML := `<?xml version="1.0"?>
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
<item>
<time>Sat Sep 30 20:11:32 IST 2023</time>
<url><![CDATA[http://example.com/path1]]></url>
<host ip="127.0.0.1">example.com</host>
<port>80</port>
<protocol>http</protocol>
<method><![CDATA[GET]]></method>
<path><![CDATA[/path1]]></path>
<extension>null</extension>
<request base64="true"><![CDATA[R0VUIC8gSFRUUC8xLjE=]]></request>
<status>200</status>
<responselength>100</responselength>
<mimetype>HTML</mimetype>
<response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
<comment></comment>
</item>
<item>
<time>Sat Sep 30 20:08:54 IST 2023</time>
<url><![CDATA[https://example.com/path2]]></url>
<host ip="127.0.0.1">example.com</host>
<port>443</port>
<protocol>https</protocol>
<method><![CDATA[POST]]></method>
<path><![CDATA[/path2]]></path>
<extension>null</extension>
<request base64="true"><![CDATA[UE9TVCAvIEhUVFAvMS4x]]></request>
<status>200</status>
<responselength>100</responselength>
<mimetype>JSON</mimetype>
<response base64="true"><![CDATA[SFRUUC8xLjEgMjAwIE9L]]></response>
<comment></comment>
</item>
</items>`
b := NewBurpFormat()
var urls []string
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
urls = append(urls, url)
return true
})
if err != nil {
t.Fatalf("Parse returned error: %v", err)
}
if len(urls) != 2 {
t.Errorf("Expected 2 URLs, got %d", len(urls))
}
expectedURLs := []string{"http://example.com/path1", "https://example.com/path2"}
if len(urls) != len(expectedURLs) {
t.Fatalf("Expected %d URLs, got %d: %v", len(expectedURLs), len(urls), urls)
}
for i, expected := range expectedURLs {
if urls[i] != expected {
t.Errorf("Expected URL %d to be '%s', got '%s'", i, expected, urls[i])
}
}
}
func TestBurpFormat_ParseEmpty(t *testing.T) {
burpXML := `<?xml version="1.0"?>
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
</items>`
b := NewBurpFormat()
var urls []string
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
urls = append(urls, url)
return true
})
if err != nil {
t.Fatalf("Parse returned error: %v", err)
}
if len(urls) != 0 {
t.Errorf("Expected 0 URLs, got %d", len(urls))
}
}
func TestBurpFormat_ParseStopEarly(t *testing.T) {
burpXML := `<?xml version="1.0"?>
<items burpVersion="2023.10.1.2" exportTime="Sat Sep 30 20:11:44 IST 2023">
<item>
<url><![CDATA[http://example.com/1]]></url>
<host>example.com</host>
<port>80</port>
<protocol>http</protocol>
<method><![CDATA[GET]]></method>
<path><![CDATA[/1]]></path>
<extension>null</extension>
<request base64="true"><![CDATA[R0VUIC8=]]></request>
<status>200</status>
<responselength>100</responselength>
<mimetype>HTML</mimetype>
<response base64="true"><![CDATA[T0s=]]></response>
<comment></comment>
</item>
<item>
<url><![CDATA[http://example.com/2]]></url>
<host>example.com</host>
<port>80</port>
<protocol>http</protocol>
<method><![CDATA[GET]]></method>
<path><![CDATA[/2]]></path>
<extension>null</extension>
<request base64="true"><![CDATA[R0VUIC8=]]></request>
<status>200</status>
<responselength>100</responselength>
<mimetype>HTML</mimetype>
<response base64="true"><![CDATA[T0s=]]></response>
<comment></comment>
</item>
</items>`
b := NewBurpFormat()
var urls []string
err := b.Parse(strings.NewReader(burpXML), func(url string) bool {
urls = append(urls, url)
return false // stop after first
})
if err != nil {
t.Fatalf("Parse returned error: %v", err)
}
if len(urls) != 1 {
t.Errorf("Expected 1 URL (stopped early), got %d", len(urls))
}
}
func TestBurpFormat_ParseMalformed(t *testing.T) {
malformedXML := `<?xml version="1.0"?>
<items burpVersion="2023.10.1.2">
<item>
<url><![CDATA[http://example.com/path1]]></url>
<!-- missing closing tags -->
</items>`
b := NewBurpFormat()
err := b.Parse(strings.NewReader(malformedXML), func(url string) bool {
return true
})
if err == nil {
t.Error("Expected error for malformed XML, got nil")
}
}
common/inputformats/formats.go
+41
View File
@@ -0,0 +1,41 @@
// TODO: This package should be abstracted out to projectdiscovery/utils
// so it can be shared between httpx, nuclei, and other tools.
package inputformats
import (
"io"
"strings"
)
// Format is an interface implemented by all input formats
type Format interface {
// Name returns the name of the format
Name() string
// Parse parses the input and calls the provided callback
// function for each URL it discovers.
Parse(input io.Reader, callback func(url string) bool) error
}
// Supported formats
var formats = []Format{
NewBurpFormat(),
}
// GetFormat returns the format by name
func GetFormat(name string) Format {
for _, f := range formats {
if strings.EqualFold(f.Name(), name) {
return f
}
}
return nil
}
// SupportedFormats returns a comma-separated list of supported format names
func SupportedFormats() string {
var names []string
for _, f := range formats {
names = append(names, f.Name())
}
return strings.Join(names, ", ")
}
common/inputformats/formats_test.go
+43
View File
@@ -0,0 +1,43 @@
package inputformats
import (
"strings"
"testing"
)
func TestGetFormat(t *testing.T) {
tests := []struct {
name string
input string
wantNil bool
wantName string
}{
{"burp lowercase", "burp", false, "burp"},
{"burp uppercase", "BURP", false, "burp"},
{"burp mixed case", "Burp", false, "burp"},
{"invalid format", "invalid", true, ""},
{"empty string", "", true, ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GetFormat(tt.input)
if tt.wantNil && got != nil {
t.Errorf("GetFormat(%q) = %v, want nil", tt.input, got)
}
if !tt.wantNil && got == nil {
t.Errorf("GetFormat(%q) = nil, want non-nil", tt.input)
}
if !tt.wantNil && got != nil && got.Name() != tt.wantName {
t.Errorf("GetFormat(%q).Name() = %q, want %q", tt.input, got.Name(), tt.wantName)
}
})
}
}
func TestSupportedFormats(t *testing.T) {
supported := SupportedFormats()
if !strings.Contains(supported, "burp") {
t.Errorf("SupportedFormats() = %q, expected to contain 'burp'", supported)
}
}
go.mod
+36 -34
View File
@@ -19,42 +19,45 @@ require (
github.com/miekg/dns v1.1.68 // indirect
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/asnmap v1.1.1
github.com/projectdiscovery/cdncheck v1.2.15
github.com/projectdiscovery/cdncheck v1.2.21
github.com/projectdiscovery/clistats v0.1.1
github.com/projectdiscovery/dsl v0.8.9
github.com/projectdiscovery/fastdialer v0.5.0
github.com/projectdiscovery/dsl v0.8.13
github.com/projectdiscovery/fastdialer v0.5.4
github.com/projectdiscovery/fdmax v0.0.4
github.com/projectdiscovery/goconfig v0.0.1
github.com/projectdiscovery/goflags v0.1.74
github.com/projectdiscovery/gologger v1.1.65
github.com/projectdiscovery/hmap v0.0.98
github.com/projectdiscovery/gologger v1.1.68
github.com/projectdiscovery/hmap v0.0.100
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/networkpolicy v0.1.32
github.com/projectdiscovery/ratelimit v0.0.82
github.com/projectdiscovery/networkpolicy v0.1.34
github.com/projectdiscovery/ratelimit v0.0.83
github.com/projectdiscovery/rawhttp v0.1.90
github.com/projectdiscovery/retryablehttp-go v1.3.0
github.com/projectdiscovery/retryablehttp-go v1.3.6
github.com/projectdiscovery/tlsx v1.2.2
github.com/projectdiscovery/useragent v0.0.105
github.com/projectdiscovery/utils v0.8.0
github.com/projectdiscovery/wappalyzergo v0.2.60
github.com/projectdiscovery/useragent v0.0.107
github.com/projectdiscovery/utils v0.9.0
github.com/projectdiscovery/wappalyzergo v0.2.66
github.com/rs/xid v1.6.0
github.com/spaolacci/murmur3 v1.1.0
github.com/stretchr/testify v1.11.1
github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77
go.etcd.io/bbolt v1.4.0 // indirect
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20250911091902-df9299821621
golang.org/x/net v0.48.0
golang.org/x/sys v0.39.0 // indirect
golang.org/x/text v0.32.0
golang.org/x/net v0.49.0
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0
)
require (
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0
github.com/dustin/go-humanize v1.0.1
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/go-viper/mapstructure/v2 v2.5.0
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
github.com/weppos/publicsuffix-go v0.50.1
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193
github.com/seh-msft/burpxml v1.0.1
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2
gopkg.in/yaml.v3 v3.0.1
)
require (
@@ -63,16 +66,16 @@ require (
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/STARRY-S/zip v0.2.1 // indirect
github.com/STARRY-S/zip v0.2.3 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.6.0 // indirect
github.com/bodgit/sevenzip v1.6.1 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/brianvoe/gofakeit/v7 v7.2.1 // indirect
github.com/charmbracelet/glamour v0.8.0 // indirect
@@ -101,14 +104,12 @@ require (
github.com/gorilla/css v1.0.1 // indirect
github.com/gosimple/slug v1.15.0 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/iangcarroll/cookiemonster v1.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kataras/jwt v0.1.10 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kljensen/snowball v0.8.0 // indirect
github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
@@ -117,22 +118,24 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mholt/archives v0.1.0 // indirect
github.com/mholt/archives v0.1.5 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/nwaples/rardecode/v2 v2.2.2 // indirect
github.com/pierrec/lz4/v4 v4.1.23 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/gostruct v0.0.2 // indirect
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
github.com/projectdiscovery/retryabledns v1.0.111 // indirect
github.com/projectdiscovery/retryabledns v1.0.113 // indirect
github.com/refraction-networking/utls v1.7.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
@@ -140,9 +143,9 @@ require (
github.com/sashabaranov/go-openai v1.37.0 // indirect
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sorairolake/lzip-go v0.3.5 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/tidwall/buntdb v1.3.1 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
@@ -167,13 +170,12 @@ require (
github.com/zcalusic/sysinfo v1.0.2 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/crypto v0.47.0 // indirect
golang.org/x/mod v0.31.0 // indirect
golang.org/x/oauth2 v0.28.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/term v0.39.0 // indirect
golang.org/x/time v0.11.0 // indirect
golang.org/x/tools v0.39.0 // indirect
golang.org/x/tools v0.40.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
+78 -72
View File
@@ -36,8 +36,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0g
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78=
@@ -48,8 +48,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46
github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
@@ -66,8 +66,8 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A=
github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc=
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8=
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/brianvoe/gofakeit/v7 v7.2.1 h1:AGojgaaCdgq4Adzrd2uWdbGNDyX6MWNhHdQBraNfOHI=
@@ -141,8 +141,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
@@ -208,11 +208,6 @@ github.com/gosimple/slug v1.15.0 h1:wRZHsRrRcs6b0XnxMUBM6WK1U1Vg5B0R7VkIf1Xzobo=
github.com/gosimple/slug v1.15.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ=
github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o=
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -241,8 +236,8 @@ github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ=
github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -275,13 +270,17 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY=
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8=
github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q=
github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A=
github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -297,8 +296,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU=
github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -310,8 +309,8 @@ github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU=
github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -321,16 +320,18 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng=
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.2.15 h1:07ati3QcI65HEWP6GgHoTtit6cfu5InEx87rpwoWios=
github.com/projectdiscovery/cdncheck v1.2.15/go.mod h1:Q9HNh2uzO/qFdl8C4RUWOHj3qIhxlaBFeayPQJm2zII=
github.com/projectdiscovery/cdncheck v1.2.21 h1:+y77BGCZoduX5bja2SGn4AdBXFwfOycaLnWWUIiZCBM=
github.com/projectdiscovery/cdncheck v1.2.21/go.mod h1:gpeX5OrzaC4DmeUGDcKrC7cPUXQvRGTY/Ui0XrVfdzU=
github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE=
github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
github.com/projectdiscovery/dsl v0.8.9 h1:weluy3pgLKxI0JCwjY6b1u5YEb9falhsiOCT5ouMnRw=
github.com/projectdiscovery/dsl v0.8.9/go.mod h1:hFLa5MOGMZXwF38WcwYPqm0uU6G46s+vbVWdMusdVv0=
github.com/projectdiscovery/fastdialer v0.5.0 h1:7XX11t/7kuenVM09r3+EeNgqYItkoTNVUc6XKFqZpHw=
github.com/projectdiscovery/fastdialer v0.5.0/go.mod h1:v4uDT/Sb0o83KNA+ggxHCw6u6BvJZglpGiAPIufaQ84=
github.com/projectdiscovery/dsl v0.8.13 h1:HjjHta7c02saH2tUGs8CN5vDeE2MyWvCV32koT8ZCWs=
github.com/projectdiscovery/dsl v0.8.13/go.mod h1:hgFaXhz/JuO+HqIXqBqYIR3ntPnqTo38MJJAzb5tIbg=
github.com/projectdiscovery/fastdialer v0.5.4 h1:+0oesDDqZcIPE5bNDmm/Xm9Xm3yjnhl4xwP+h5D1TE4=
github.com/projectdiscovery/fastdialer v0.5.4/go.mod h1:KCzt6WnSAj9umiUBRCaC0EJSEyeshxDoowfwjxodmQw=
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -339,36 +340,36 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ
github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw=
github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c=
github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4=
github.com/projectdiscovery/gologger v1.1.65 h1:tVIjEke+UaQ3b2dj89AHCHGgeyUYIO0E35Yu+ppU+bk=
github.com/projectdiscovery/gologger v1.1.65/go.mod h1:myCMJMPu1TDl48LQGj+w7MhOdVKYAZNSOURHhK5XaaM=
github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU=
github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg=
github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU=
github.com/projectdiscovery/hmap v0.0.100 h1:DBZ3Req9lWf4P1YC9PRa4eiMvLY0Uxud43NRBcocPfs=
github.com/projectdiscovery/hmap v0.0.100/go.mod h1:2O06pR8pHOP9wSmxAoxuM45U7E+UqOqOdlSIeddM0bA=
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q=
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w=
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/networkpolicy v0.1.32 h1:MD561zl0/fSzcNTlDlf/7pcQbYzulYoQssFBFjb2pcg=
github.com/projectdiscovery/networkpolicy v0.1.32/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU=
github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak=
github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM=
github.com/projectdiscovery/networkpolicy v0.1.34 h1:TRwNbgMwdx3NC190TKSLwtTvr0JAIZAlnWkOhW0yBME=
github.com/projectdiscovery/networkpolicy v0.1.34/go.mod h1:GJ20E7fJoA2vk8ZBSa1Cvc5WyP8RxglF5bZmYgK8jag=
github.com/projectdiscovery/ratelimit v0.0.83 h1:hfb36QvznBrjA4FNfpFE8AYRVBYrfJh8qHVROLQgl54=
github.com/projectdiscovery/ratelimit v0.0.83/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM=
github.com/projectdiscovery/rawhttp v0.1.90 h1:LOSZ6PUH08tnKmWsIwvwv1Z/4zkiYKYOSZ6n+8RFKtw=
github.com/projectdiscovery/rawhttp v0.1.90/go.mod h1:VZYAM25UI/wVB3URZ95ZaftgOnsbphxyAw/XnQRRz4Y=
github.com/projectdiscovery/retryabledns v1.0.111 h1:iyMdCDgNmaSRJYcGqB+SLlvlw9WijlbJ6Q9OEpRAWsQ=
github.com/projectdiscovery/retryabledns v1.0.111/go.mod h1:6TOPJ3QAE4reBu6bvsGsTcyEb+OypcKYFQH7yVsjyIM=
github.com/projectdiscovery/retryablehttp-go v1.3.0 h1:ro/IYNx4fSdz0hFCFIpQ9++SlOAa4WdQS1i6GMUgSEk=
github.com/projectdiscovery/retryablehttp-go v1.3.0/go.mod h1:B+fiMihnqSxCKM0SoEPu92u6FL5hVBfmd6j3SwrFpY0=
github.com/projectdiscovery/retryabledns v1.0.113 h1:s+DAzdJ8XhLxRgt5636H0HG9OqHsGRjX9wTrLSTMqlQ=
github.com/projectdiscovery/retryabledns v1.0.113/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA=
github.com/projectdiscovery/retryablehttp-go v1.3.6 h1:dLb0/YVX+oX70gpWxN5GXT8pCKpn8fdXfwOq2TsXxNY=
github.com/projectdiscovery/retryablehttp-go v1.3.6/go.mod h1:tKVxmL4ixWy1MjYk5GJvFL0Cp10fnQgSp2F6bSBEypI=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.2.2 h1:Y96QBqeD2anpzEtBl4kqNbwzXh2TrzJuXfgiBLvK+SE=
github.com/projectdiscovery/tlsx v1.2.2/go.mod h1:ZJl9F1sSl0sdwE+lR0yuNHVX4Zx6tCSTqnNxnHCFZB4=
github.com/projectdiscovery/useragent v0.0.105 h1:yFGFTfWZ/RZP5XbGRJtvKcbRNnlGI6xfPRXHb8DWdhg=
github.com/projectdiscovery/useragent v0.0.105/go.mod h1:jWG1BD2yu8EC3olt6Amke7BiyLkXzpErI7Jtzr/tWZM=
github.com/projectdiscovery/utils v0.8.0 h1:8d79OCs5xGDNXdKxMUKMY/lgQSUWJMYB1B2Sx+oiqkQ=
github.com/projectdiscovery/utils v0.8.0/go.mod h1:CU6tjtyTRxBrnNek+GPJplw4IIHcXNZNKO09kWgqTdg=
github.com/projectdiscovery/wappalyzergo v0.2.60 h1:CGpQy2qHxOBQSikdNjZomHUP9fVljKBfDJeodpjVp7A=
github.com/projectdiscovery/wappalyzergo v0.2.60/go.mod h1:8FtSVcmPRZU0g1euBpdSYEBHIvB7Zz9MOb754ZqZmfU=
github.com/projectdiscovery/useragent v0.0.107 h1:45gSBda052fv2Gtxtnpx7cu2rWtUpZEQRGAoYGP6F5M=
github.com/projectdiscovery/useragent v0.0.107/go.mod h1:yv5ZZLDT/kq6P+NvBcCPq6sjEVQtZGgO+OvvHzZ+WtY=
github.com/projectdiscovery/utils v0.9.0 h1:eu9vdbP0VYXI9nGSLfnOpUqBeW9/B/iSli7U8gPKZw8=
github.com/projectdiscovery/utils v0.9.0/go.mod h1:zcVu1QTlMi5763qCol/L3ROnbd/UPSBP8fI5PmcnF6s=
github.com/projectdiscovery/wappalyzergo v0.2.66 h1:DEF7wthjvBo6oYKxfKL6vPNaqsKYUmiWODt7Mybcins=
github.com/projectdiscovery/wappalyzergo v0.2.66/go.mod h1:Oc+U2RPJObmpi6LW5lTMEDiKagcKZNkEfZfwrVMURa0=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0=
github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ=
@@ -388,6 +389,8 @@ github.com/sashabaranov/go-openai v1.37.0 h1:hQQowgYm4OXJ1Z/wTrE+XZaO20BYsL0R3uR
github.com/sashabaranov/go-openai v1.37.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/seh-msft/burpxml v1.0.1 h1:5G3QPSzvfA1WcX7LkxmKBmK2RnNyGviGWnJPumE0nwg=
github.com/seh-msft/burpxml v1.0.1/go.mod h1:lTViCHPtGGS0scK0B4krm6Ld1kVZLWzQccwUomRc58I=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
@@ -399,14 +402,18 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg=
github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk=
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -419,8 +426,6 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
@@ -454,9 +459,9 @@ github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
github.com/vulncheck-oss/go-exploit v1.51.0 h1:HTmJ4Q94tbEDPb35mQZn6qMg4rT+Sw9n+L7g3Pjr+3o=
github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8lvztnnN2wCXy7No6c=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
github.com/weppos/publicsuffix-go v0.50.1 h1:elrBHeSkS/eIb169+DnLrknqmdP4AjT0Q0tEdytz1Og=
github.com/weppos/publicsuffix-go v0.50.1/go.mod h1:znn0JVXjcR5hpUl9pbEogwH6I710rA1AX0QQPT0bf+k=
github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko=
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0=
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
@@ -494,8 +499,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t
github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk=
github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I=
github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM=
github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8=
go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
@@ -521,12 +526,12 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -559,8 +564,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -587,12 +592,12 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -653,12 +658,12 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -667,12 +672,12 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -687,9 +692,10 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
@@ -723,8 +729,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
runner/banner.go
+1 -1
View File
@@ -16,7 +16,7 @@ const banner = `
`
// Version is the current Version of httpx
const Version = `v1.7.4`
const Version = `v1.8.1`
// showBanner is used to show the banner to the user
func showBanner() {
runner/cpe.go
+225
View File
@@ -0,0 +1,225 @@
package runner
import (
"encoding/json"
"fmt"
"strings"
awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
)
type CPEInfo struct {
Product string `json:"product,omitempty"`
Vendor string `json:"vendor,omitempty"`
CPE string `json:"cpe,omitempty"`
}
type CPEDetector struct {
titlePatterns map[string][]CPEInfo
bodyPatterns map[string][]CPEInfo
faviconPatterns map[string][]CPEInfo
}
type rawQuery struct {
Name string `json:"name"`
Vendor json.RawMessage `json:"vendor"`
Type string `json:"type"`
Engines []rawEngine `json:"engines"`
}
type rawEngine struct {
Platform string `json:"platform"`
Queries []string `json:"queries"`
}
func NewCPEDetector() (*CPEDetector, error) {
data, err := awesomesearchqueries.GetQueries()
if err != nil {
return nil, fmt.Errorf("failed to load queries: %w", err)
}
var queries []rawQuery
if err := json.Unmarshal(data, &queries); err != nil {
return nil, fmt.Errorf("failed to parse queries: %w", err)
}
detector := &CPEDetector{
titlePatterns: make(map[string][]CPEInfo),
bodyPatterns: make(map[string][]CPEInfo),
faviconPatterns: make(map[string][]CPEInfo),
}
for _, q := range queries {
vendor := parseVendor(q.Vendor)
info := CPEInfo{
Product: q.Name,
Vendor: vendor,
CPE: generateCPE(vendor, q.Name),
}
for _, engine := range q.Engines {
for _, query := range engine.Queries {
detector.extractPattern(query, info)
}
}
}
return detector, nil
}
func parseVendor(raw json.RawMessage) string {
var vendorStr string
if err := json.Unmarshal(raw, &vendorStr); err == nil {
return vendorStr
}
var vendorSlice []string
if err := json.Unmarshal(raw, &vendorSlice); err == nil && len(vendorSlice) > 0 {
return vendorSlice[0]
}
return ""
}
func generateCPE(vendor, product string) string {
if vendor == "" || product == "" {
return ""
}
return fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*",
strings.ToLower(strings.ReplaceAll(vendor, " ", "_")),
strings.ToLower(strings.ReplaceAll(product, " ", "_")))
}
func (d *CPEDetector) extractPattern(query string, info CPEInfo) {
query = strings.TrimSpace(query)
titlePrefixes := []string{
"http.title:",
"title=",
"title==",
"intitle:",
"title:",
"title='",
`title="`,
}
for _, prefix := range titlePrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
pattern = strings.TrimPrefix(pattern, prefix[:len(prefix)-1])
if pattern != "" {
pattern = strings.ToLower(pattern)
d.titlePatterns[pattern] = appendUnique(d.titlePatterns[pattern], info)
}
return
}
}
bodyPrefixes := []string{
"http.html:",
"body=",
"body==",
"intext:",
}
for _, prefix := range bodyPrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
if pattern != "" {
pattern = strings.ToLower(pattern)
d.bodyPatterns[pattern] = appendUnique(d.bodyPatterns[pattern], info)
}
return
}
}
faviconPrefixes := []string{
"http.favicon.hash:",
"icon_hash=",
"icon_hash==",
}
for _, prefix := range faviconPrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
if pattern != "" {
d.faviconPatterns[pattern] = appendUnique(d.faviconPatterns[pattern], info)
}
return
}
}
}
func extractQuotedValue(s string) string {
s = strings.TrimSpace(s)
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
if idx := strings.Index(s, "\" ||"); idx > 0 {
s = s[:idx]
}
if idx := strings.Index(s, "' ||"); idx > 0 {
s = s[:idx]
}
return strings.TrimSpace(s)
}
func appendUnique(slice []CPEInfo, info CPEInfo) []CPEInfo {
for _, existing := range slice {
if existing.Product == info.Product && existing.Vendor == info.Vendor {
return slice
}
}
return append(slice, info)
}
func (d *CPEDetector) Detect(title, body, faviconHash string) []CPEInfo {
seen := make(map[string]bool)
var results []CPEInfo
titleLower := strings.ToLower(title)
bodyLower := strings.ToLower(body)
for pattern, infos := range d.titlePatterns {
if strings.Contains(titleLower, pattern) {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
for pattern, infos := range d.bodyPatterns {
if strings.Contains(bodyLower, pattern) {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
if faviconHash != "" {
if infos, ok := d.faviconPatterns[faviconHash]; ok {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
return results
}
runner/options.go
+25
View File
@@ -23,6 +23,7 @@ import (
customport "github.com/projectdiscovery/httpx/common/customports"
fileutilz "github.com/projectdiscovery/httpx/common/fileutil"
httpxcommon "github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/httpx/common/inputformats"
"github.com/projectdiscovery/httpx/common/stringz"
"github.com/projectdiscovery/networkpolicy"
pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
@@ -85,6 +86,8 @@ type ScanOptions struct {
NoFallback bool
NoFallbackScheme bool
TechDetect bool
CPEDetect bool
WordPress bool
StoreChain bool
StoreVisionReconClusters bool
MaxResponseBodySizeToSave int
@@ -148,6 +151,8 @@ func (s *ScanOptions) Clone() *ScanOptions {
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
CPEDetect: s.CPEDetect,
WordPress: s.WordPress,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
@@ -187,6 +192,7 @@ type Options struct {
SocksProxy string
Proxy string
InputFile string
InputMode string
InputTargetHost goflags.StringSlice
Methods string
RequestURI string
@@ -257,6 +263,8 @@ type Options struct {
NoFallback bool
NoFallbackScheme bool
TechDetect bool
CPEDetect bool
WordPress bool
CustomFingerprintFile string
TLSGrab bool
protocol string
@@ -346,6 +354,8 @@ type Options struct {
// AssetFileUpload
AssetFileUpload string
TeamID string
// SecretFile is the path to the secret file for authentication
SecretFile string
// OnClose adds a callback function that is invoked when httpx is closed
// to be exact at end of existing closures
OnClose func()
@@ -370,6 +380,7 @@ func ParseOptions() *Options {
flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"),
flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"),
flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions),
flagSet.StringVarP(&options.InputMode, "input-mode", "im", "", fmt.Sprintf("mode of input file (%s)", inputformats.SupportedFormats())),
)
flagSet.CreateGroup("Probes", "Probes",
@@ -388,6 +399,8 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"),
flagSet.BoolVar(&options.CPEDetect, "cpe", false, "display CPE (Common Platform Enumeration) based on awesome-search-queries"),
flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress plugins and themes"),
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
flagSet.BoolVarP(&options.OutputWebSocket, "websocket", "ws", false, "display server using websocket"),
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
@@ -517,6 +530,7 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"),
flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"),
)
flagSet.CreateGroup("debug", "Debug",
@@ -671,6 +685,17 @@ func (options *Options) ValidateOptions() error {
return fmt.Errorf("file '%s' does not exist", options.InputRawRequest)
}
if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) {
return fmt.Errorf("secret file '%s' does not exist", options.SecretFile)
}
if options.InputMode != "" && inputformats.GetFormat(options.InputMode) == nil {
return fmt.Errorf("invalid input mode '%s', supported formats: %s", options.InputMode, inputformats.SupportedFormats())
}
if options.InputMode != "" && options.InputFile == "" {
return errors.New("-im/-input-mode requires -l/-list to specify an input file")
}
if options.Silent {
incompatibleFlagsList := flagsIncompatibleWithSilent(options)
if len(incompatibleFlagsList) > 0 {
runner/runner.go
+253 -41
View File
@@ -35,7 +35,9 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/hashes/jarm"
"github.com/projectdiscovery/httpx/common/inputformats"
"github.com/projectdiscovery/httpx/common/pagetypeclassifier"
"github.com/projectdiscovery/httpx/common/authprovider"
"github.com/projectdiscovery/httpx/static"
"github.com/projectdiscovery/mapcidr/asn"
"github.com/projectdiscovery/networkpolicy"
@@ -81,6 +83,8 @@ type Runner struct {
options *Options
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
cpeDetector *CPEDetector
wpDetector *WordPressDetector
scanopts ScanOptions
hm *hybrid.HybridMap
excludeCdn bool
@@ -92,6 +96,7 @@ type Runner struct {
pHashClusters []pHashCluster
simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection
httpApiEndpoint *Server
authProvider authprovider.AuthProvider
}
func (r *Runner) HTTPX() *httpx.HTTPX {
@@ -133,9 +138,26 @@ func New(options *Options) (*Runner, error) {
return nil, errors.Wrap(err, "could not create wappalyzer client")
}
if options.CPEDetect || options.JSONOutput || options.CSVOutput {
runner.cpeDetector, err = NewCPEDetector()
if err != nil {
gologger.Warning().Msgf("Could not create CPE detector: %s", err)
}
}
if options.WordPress || options.JSONOutput || options.CSVOutput {
runner.wpDetector, err = NewWordPressDetector()
if err != nil {
gologger.Warning().Msgf("Could not create WordPress detector: %s", err)
}
}
if options.StoreResponseDir != "" {
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
// Don't remove index files if skip-dedupe is enabled (we want to append, not truncate)
if !options.SkipDedupe {
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
}
}
httpxOptions := httpx.DefaultOptions
@@ -297,6 +319,8 @@ func New(options *Options) (*Runner, error) {
scanopts.NoFallback = options.NoFallback
scanopts.NoFallbackScheme = options.NoFallbackScheme
scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload
scanopts.CPEDetect = options.CPEDetect || options.JSONOutput || options.CSVOutput
scanopts.WordPress = options.WordPress || options.JSONOutput || options.CSVOutput
scanopts.StoreChain = options.StoreChain
scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
@@ -391,6 +415,16 @@ func New(options *Options) (*Runner, error) {
}
runner.pageTypeClassifier = pageTypeClassifier
if options.SecretFile != "" {
authProviderOpts := &authprovider.AuthProviderOptions{
SecretsFiles: []string{options.SecretFile},
}
runner.authProvider, err = authprovider.NewAuthProvider(authProviderOpts)
if err != nil {
return nil, errors.Wrap(err, "could not create auth provider")
}
}
if options.HttpApiEndpoint != "" {
apiServer := NewServer(options.HttpApiEndpoint, options)
gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint)
@@ -481,27 +515,44 @@ func (r *Runner) prepareInputPaths() {
}
}
var duplicateTargetErr = errors.New("duplicate target")
func (r *Runner) prepareInput() {
var numHosts int
// check if input target host(s) have been provided
if len(r.options.InputTargetHost) > 0 {
for _, target := range r.options.InputTargetHost {
expandedTarget, _ := r.countTargetFromRawTarget(target)
if expandedTarget > 0 {
expandedTarget, err := r.countTargetFromRawTarget(target)
if err == nil && expandedTarget > 0 {
numHosts += expandedTarget
r.hm.Set(target, nil) //nolint
r.hm.Set(target, []byte("1")) //nolint
} else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) {
if v, ok := r.hm.Get(target); ok {
cnt, _ := strconv.Atoi(string(v))
_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
numHosts += 1
}
}
}
}
// check if file has been provided
if fileutil.FileExists(r.options.InputFile) {
finput, err := os.Open(r.options.InputFile)
if err != nil {
gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
}
numHosts, err = r.loadAndCloseFile(finput)
if err != nil {
gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
// check if input mode is specified for special format handling
if format := r.getInputFormat(); format != nil {
numTargets, err := r.loadFromFormat(r.options.InputFile, format)
if err != nil {
gologger.Fatal().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err)
}
numHosts = numTargets
} else {
finput, err := os.Open(r.options.InputFile)
if err != nil {
gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
}
numHosts, err = r.loadAndCloseFile(finput)
if err != nil {
gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err)
}
}
} else if r.options.InputFile != "" {
files, err := fileutilz.ListFilesWithPattern(r.options.InputFile)
@@ -595,19 +646,52 @@ func (r *Runner) testAndSet(k string) bool {
return true
}
// getInputFormat returns the format for the configured input mode.
// Returns nil if no input mode is configured, or logs fatal if the format is invalid.
func (r *Runner) getInputFormat() inputformats.Format {
if r.options.InputMode == "" {
return nil
}
format := inputformats.GetFormat(r.options.InputMode)
if format == nil {
gologger.Fatal().Msgf("Invalid input mode '%s'. Supported: %s\n", r.options.InputMode, inputformats.SupportedFormats())
}
return format
}
func (r *Runner) streamInput() (chan string, error) {
out := make(chan string)
go func() {
defer close(out)
if fileutil.FileExists(r.options.InputFile) {
fchan, err := fileutil.ReadFile(r.options.InputFile)
if err != nil {
return
}
for item := range fchan {
if r.options.SkipDedupe || r.testAndSet(item) {
out <- item
// check if input mode is specified for special format handling
if format := r.getInputFormat(); format != nil {
finput, err := os.Open(r.options.InputFile)
if err != nil {
gologger.Error().Msgf("Could not open input file '%s': %s\n", r.options.InputFile, err)
return
}
defer finput.Close()
if err := format.Parse(finput, func(item string) bool {
item = strings.TrimSpace(item)
if r.options.SkipDedupe || r.testAndSet(item) {
out <- item
}
return true
}); err != nil {
gologger.Error().Msgf("Could not parse input file '%s': %s\n", r.options.InputFile, err)
return
}
} else {
fchan, err := fileutil.ReadFile(r.options.InputFile)
if err != nil {
return
}
for item := range fchan {
if r.options.SkipDedupe || r.testAndSet(item) {
out <- item
}
}
}
} else if r.options.InputFile != "" {
@@ -647,22 +731,54 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) {
for scanner.Scan() {
target := strings.TrimSpace(scanner.Text())
// Used just to get the exact number of targets
expandedTarget, _ := r.countTargetFromRawTarget(target)
if expandedTarget > 0 {
expandedTarget, err := r.countTargetFromRawTarget(target)
if err == nil && expandedTarget > 0 {
numTargets += expandedTarget
r.hm.Set(target, nil) //nolint
r.hm.Set(target, []byte("1")) //nolint
} else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) {
if v, ok := r.hm.Get(target); ok {
cnt, _ := strconv.Atoi(string(v))
_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
numTargets += 1
}
}
}
err = finput.Close()
return numTargets, err
}
func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) {
finput, err := os.Open(filePath)
if err != nil {
return 0, err
}
defer finput.Close()
err = format.Parse(finput, func(target string) bool {
target = strings.TrimSpace(target)
expandedTarget, countErr := r.countTargetFromRawTarget(target)
if countErr == nil && expandedTarget > 0 {
numTargets += expandedTarget
r.hm.Set(target, []byte("1")) //nolint
} else if r.options.SkipDedupe && errors.Is(countErr, duplicateTargetErr) {
if v, ok := r.hm.Get(target); ok {
cnt, _ := strconv.Atoi(string(v))
_ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1)))
numTargets += 1
}
}
return true
})
return numTargets, err
}
func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err error) {
if rawTarget == "" {
return 0, nil
}
if _, ok := r.hm.Get(rawTarget); ok {
return 0, nil
return 0, duplicateTargetErr
}
expandedTarget := 0
@@ -909,7 +1025,8 @@ func (r *Runner) RunEnumeration() {
gologger.Fatal().Msgf("Could not create response directory '%s': %s\n", responseDirPath, err)
}
indexPath := filepath.Join(responseDirPath, "index.txt")
if r.options.Resume {
// Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd)
if r.options.Resume || r.options.SkipDedupe {
indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
indexFile, err = os.Create(indexPath)
@@ -923,7 +1040,8 @@ func (r *Runner) RunEnumeration() {
if r.options.Screenshot {
var err error
indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt")
if r.options.Resume {
// Append if resume is enabled or skip-dedupe is enabled (never truncate with -sd)
if r.options.Resume || r.options.SkipDedupe {
indexScreenshotFile, err = os.OpenFile(indexScreenshotPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
} else {
indexScreenshotFile, err = os.Create(indexScreenshotPath)
@@ -1115,10 +1233,8 @@ func (r *Runner) RunEnumeration() {
// store responses or chain in directory
if resp.Err == nil {
URL, _ := urlutil.Parse(resp.URL)
domainFile := resp.Method + ":" + URL.EscapedString()
hash := hashes.Sha1([]byte(domainFile))
domainResponseFile := fmt.Sprintf("%s.txt", hash)
screenshotResponseFile := fmt.Sprintf("%s.png", hash)
domainResponseFile := fmt.Sprintf("%s.txt", resp.FileNameHash)
screenshotResponseFile := fmt.Sprintf("%s.png", resp.FileNameHash)
hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response")
domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot")
@@ -1340,14 +1456,28 @@ func (r *Runner) RunEnumeration() {
}
}
if len(r.options.requestURIs) > 0 {
for _, p := range r.options.requestURIs {
scanopts := r.scanopts.Clone()
scanopts.RequestURI = p
r.process(k, wg, r.hp, protocol, scanopts, output)
runProcess := func(times int) {
for i := 0; i < times; i++ {
if len(r.options.requestURIs) > 0 {
for _, p := range r.options.requestURIs {
scanopts := r.scanopts.Clone()
scanopts.RequestURI = p
r.process(k, wg, r.hp, protocol, scanopts, output)
}
} else {
r.process(k, wg, r.hp, protocol, &r.scanopts, output)
}
}
} else {
r.process(k, wg, r.hp, protocol, &r.scanopts, output)
}
if r.options.Stream {
runProcess(1)
} else if v, ok := r.hm.Get(k); ok {
cnt, err := strconv.Atoi(string(v))
if err != nil || cnt <= 0 {
cnt = 1
}
runProcess(cnt)
}
return nil
@@ -1600,7 +1730,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
results <- httpx.Target{Host: target}
return
}
ips, _, _, err := getDNSData(hp, URL.Host)
ips, _, _, err := getDNSData(hp, URL.Hostname())
if err != nil || len(ips) == 0 {
results <- httpx.Target{Host: target}
return
@@ -1693,6 +1823,16 @@ retry:
}
hp.SetCustomHeaders(req, hp.CustomHeaders)
// Apply auth strategies if auth provider is configured
if r.authProvider != nil {
if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 {
for _, strategy := range strategies {
strategy.ApplyOnRR(req)
}
}
}
// We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size)
if scanopts.RequestBody != "" {
req.ContentLength = int64(len(scanopts.RequestBody))
@@ -2239,7 +2379,7 @@ retry:
domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response")
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
var responsePath string
var responsePath, fileNameHash string
// store response
if scanopts.StoreResponse || scanopts.StoreChain {
if r.options.OmitBody {
@@ -2260,9 +2400,33 @@ retry:
data = append(data, []byte("\n\n\n")...)
data = append(data, []byte(fullURL)...)
_ = fileutil.CreateFolder(responseBaseDir)
writeErr := os.WriteFile(responsePath, data, 0644)
if writeErr != nil {
gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
basePath := strings.TrimSuffix(responsePath, ".txt")
var idx int
for idx = 0; ; idx++ {
targetPath := responsePath
if idx > 0 {
targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx)
}
f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err == nil {
_, writeErr := f.Write(data)
_ = f.Close()
if writeErr != nil {
gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr)
}
break
}
if !os.IsExist(err) {
gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err)
break
}
}
if idx == 0 {
fileNameHash = hash
} else {
fileNameHash = fmt.Sprintf("%s_%d", hash, idx)
}
}
@@ -2349,6 +2513,47 @@ retry:
}
}
var cpeMatches []CPEInfo
if r.cpeDetector != nil {
cpeMatches = r.cpeDetector.Detect(title, string(resp.Data), faviconMMH3)
if len(cpeMatches) > 0 && r.options.CPEDetect {
for _, cpe := range cpeMatches {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Cyan(cpe.CPE).String())
} else {
builder.WriteString(cpe.CPE)
}
builder.WriteRune(']')
}
}
}
var wpInfo *WordPressInfo
if r.wpDetector != nil {
wpInfo = r.wpDetector.Detect(string(resp.Data))
if wpInfo.HasData() && r.options.WordPress {
if len(wpInfo.Plugins) > 0 {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Green("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")).String())
} else {
builder.WriteString("wp-plugins:" + strings.Join(wpInfo.Plugins, ","))
}
builder.WriteRune(']')
}
if len(wpInfo.Themes) > 0 {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Green("wp-themes:" + strings.Join(wpInfo.Themes, ",")).String())
} else {
builder.WriteString("wp-themes:" + strings.Join(wpInfo.Themes, ","))
}
builder.WriteRune(']')
}
}
}
result := Result{
Timestamp: time.Now(),
Request: request,
@@ -2412,6 +2617,9 @@ retry:
RequestRaw: requestDump,
Response: resp,
FaviconData: faviconData,
FileNameHash: fileNameHash,
CPE: cpeMatches,
WordPress: wpInfo,
}
if resp.BodyDomains != nil {
result.Fqdns = resp.BodyDomains.Fqdns
@@ -2531,6 +2739,10 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
}
clone.SetURL(resolvedURL)
// Update Host header to match resolved URL host (important after redirects)
if resolvedURL.Host != "" && resolvedURL.Host != clone.Host {
clone.Host = resolvedURL.Host
}
respFav, err := hp.Do(clone, httpx.UnsafeOptions{})
if err != nil || len(respFav.Data) == 0 {
tries++
runner/runner_test.go
+39 -6
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/pkg/errors"
_ "github.com/projectdiscovery/fdmax/autofdmax"
"github.com/projectdiscovery/httpx/common/httpx"
"github.com/projectdiscovery/mapcidr/asn"
@@ -67,6 +68,36 @@ func TestRunner_probeall_targets(t *testing.T) {
require.ElementsMatch(t, expected, got, "could not expected output")
}
func TestRunner_probeall_targets_with_port(t *testing.T) {
options := &Options{
ProbeAllIPS: true,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
inputWithPort := "http://one.one.one.one:8080"
inputWithoutPort := "one.one.one.one"
gotWithPort := []httpx.Target{}
for target := range r.targets(r.hp, inputWithPort) {
gotWithPort = append(gotWithPort, target)
}
gotWithoutPort := []httpx.Target{}
for target := range r.targets(r.hp, inputWithoutPort) {
gotWithoutPort = append(gotWithoutPort, target)
}
require.True(t, len(gotWithPort) > 0, "probe-all-ips with port should return at least one target")
require.True(t, len(gotWithoutPort) > 0, "probe-all-ips without port should return at least one target")
require.Equal(t, len(gotWithPort), len(gotWithoutPort), "probe-all-ips should return same number of IPs with or without port")
for _, target := range gotWithPort {
require.Equal(t, inputWithPort, target.Host, "Host should be preserved with port")
require.NotEmpty(t, target.CustomIP, "CustomIP should be populated")
}
}
func TestRunner_cidr_targets(t *testing.T) {
options := &Options{}
r, err := New(options)
@@ -124,7 +155,9 @@ func TestRunner_asn_targets(t *testing.T) {
}
func TestRunner_countTargetFromRawTarget(t *testing.T) {
options := &Options{}
options := &Options{
SkipDedupe: false,
}
r, err := New(options)
require.Nil(t, err, "could not create httpx runner")
@@ -139,7 +172,7 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
err = r.hm.Set(input, nil)
require.Nil(t, err, "could not set value to hm")
got, err = r.countTargetFromRawTarget(input)
require.Nil(t, err, "could not count targets")
require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error")
require.Equal(t, expected, got, "got wrong output")
input = "173.0.84.0/24"
@@ -227,10 +260,10 @@ func TestCreateNetworkpolicyInstance_AllowDenyFlags(t *testing.T) {
runner := &Runner{}
tests := []struct {
name string
allow []string
deny []string
testCases []struct {
name string
allow []string
deny []string
testCases []struct {
ip string
expected bool
reason string
runner/wordpress.go
+118
View File
@@ -0,0 +1,118 @@
package runner
import (
"bufio"
"bytes"
"regexp"
"strings"
awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
)
type WordPressInfo struct {
Plugins []string `json:"plugins,omitempty"`
Themes []string `json:"themes,omitempty"`
}
type WordPressDetector struct {
knownPlugins map[string]struct{}
knownThemes map[string]struct{}
pluginRegex *regexp.Regexp
themeRegex *regexp.Regexp
}
func NewWordPressDetector() (*WordPressDetector, error) {
detector := &WordPressDetector{
knownPlugins: make(map[string]struct{}),
knownThemes: make(map[string]struct{}),
}
var err error
detector.pluginRegex, err = regexp.Compile(`/wp-content/plugins/([a-zA-Z0-9_-]+)/`)
if err != nil {
return nil, err
}
detector.themeRegex, err = regexp.Compile(`/wp-content/themes/([a-zA-Z0-9_-]+)/`)
if err != nil {
return nil, err
}
pluginsData, err := awesomesearchqueries.GetWordPressPlugins()
if err != nil {
return nil, err
}
if err := detector.loadList(pluginsData, detector.knownPlugins); err != nil {
return nil, err
}
themesData, err := awesomesearchqueries.GetWordPressThemes()
if err != nil {
return nil, err
}
if err := detector.loadList(themesData, detector.knownThemes); err != nil {
return nil, err
}
return detector, nil
}
func (d *WordPressDetector) loadList(data []byte, target map[string]struct{}) error {
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
target[line] = struct{}{}
}
}
return scanner.Err()
}
func (d *WordPressDetector) Detect(body string) *WordPressInfo {
if body == "" {
return nil
}
info := &WordPressInfo{}
seenPlugins := make(map[string]struct{})
seenThemes := make(map[string]struct{})
if matches := d.pluginRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 {
for _, match := range matches {
if len(match) > 1 {
plugin := match[1]
if _, seen := seenPlugins[plugin]; !seen {
if _, known := d.knownPlugins[plugin]; known {
info.Plugins = append(info.Plugins, plugin)
seenPlugins[plugin] = struct{}{}
}
}
}
}
}
if matches := d.themeRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 {
for _, match := range matches {
if len(match) > 1 {
theme := match[1]
if _, seen := seenThemes[theme]; !seen {
if _, known := d.knownThemes[theme]; known {
info.Themes = append(info.Themes, theme)
seenThemes[theme] = struct{}{}
}
}
}
}
}
if len(info.Plugins) == 0 && len(info.Themes) == 0 {
return nil
}
return info
}
func (w *WordPressInfo) HasData() bool {
return w != nil && (len(w.Plugins) > 0 || len(w.Themes) > 0)
}
snapcraft.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ name: httpx
summary: httpx is a fast and multi-purpose HTTP toolkit
description: |
httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads.
version: 'v1.7.4'
version: 'v1.8.1'
icon: static/httpx-logo.png
license: MIT
base: core18