diff --git a/common/authprovider/authx/basic_auth.go b/common/authprovider/authx/basic_auth.go new file mode 100644 index 0000000..b757906 --- /dev/null +++ b/common/authprovider/authx/basic_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BasicAuthStrategy{} +) + +// BasicAuthStrategy is a strategy for basic auth +type BasicAuthStrategy struct { + Data *Secret +} + +// NewBasicAuthStrategy creates a new basic auth strategy +func NewBasicAuthStrategy(data *Secret) *BasicAuthStrategy { + return &BasicAuthStrategy{Data: data} +} + +// Apply applies the basic auth strategy to the request +func (s *BasicAuthStrategy) Apply(req *http.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} + +// ApplyOnRR applies the basic auth strategy to the retryable request +func (s *BasicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.SetBasicAuth(s.Data.Username, s.Data.Password) +} diff --git a/common/authprovider/authx/bearer_auth.go b/common/authprovider/authx/bearer_auth.go new file mode 100644 index 0000000..edf6f43 --- /dev/null +++ b/common/authprovider/authx/bearer_auth.go @@ -0,0 +1,31 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &BearerTokenAuthStrategy{} +) + +// BearerTokenAuthStrategy is a strategy for bearer token auth +type BearerTokenAuthStrategy struct { + Data *Secret +} + +// NewBearerTokenAuthStrategy creates a new bearer token auth strategy +func NewBearerTokenAuthStrategy(data *Secret) *BearerTokenAuthStrategy { + return &BearerTokenAuthStrategy{Data: data} +} + +// Apply applies the bearer token auth strategy to the request +func (s *BearerTokenAuthStrategy) Apply(req *http.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} + +// ApplyOnRR applies the bearer token auth strategy to the retryable request +func (s *BearerTokenAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + req.Header.Set("Authorization", "Bearer "+s.Data.Token) +} diff --git a/common/authprovider/authx/cookies_auth.go b/common/authprovider/authx/cookies_auth.go new file mode 100644 index 0000000..0b94e85 --- /dev/null +++ b/common/authprovider/authx/cookies_auth.go @@ -0,0 +1,60 @@ +package authx + +import ( + "net/http" + "slices" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &CookiesAuthStrategy{} +) + +// CookiesAuthStrategy is a strategy for cookies auth +type CookiesAuthStrategy struct { + Data *Secret +} + +// NewCookiesAuthStrategy creates a new cookies auth strategy +func NewCookiesAuthStrategy(data *Secret) *CookiesAuthStrategy { + return &CookiesAuthStrategy{Data: data} +} + +// Apply applies the cookies auth strategy to the request +func (s *CookiesAuthStrategy) Apply(req *http.Request) { + for _, cookie := range s.Data.Cookies { + c := &http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + } + req.AddCookie(c) + } +} + +// ApplyOnRR applies the cookies auth strategy to the retryable request +func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + existingCookies := req.Cookies() + + for _, newCookie := range s.Data.Cookies { + for i, existing := range existingCookies { + if existing.Name == newCookie.Key { + existingCookies = slices.Delete(existingCookies, i, i+1) + break + } + } + } + + // Clear and reset remaining cookies + req.Header.Del("Cookie") + for _, cookie := range existingCookies { + req.AddCookie(cookie) + } + // Add new cookies + for _, cookie := range s.Data.Cookies { + req.AddCookie(&http.Cookie{ + Name: cookie.Key, + Value: cookie.Value, + }) + } +} diff --git a/common/authprovider/authx/file.go b/common/authprovider/authx/file.go new file mode 100644 index 0000000..a31fb9a --- /dev/null +++ b/common/authprovider/authx/file.go @@ -0,0 +1,242 @@ +package authx + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/projectdiscovery/utils/errkit" + "github.com/projectdiscovery/utils/generic" + stringsutil "github.com/projectdiscovery/utils/strings" + "gopkg.in/yaml.v3" +) + +type AuthType string + +const ( + BasicAuth AuthType = "BasicAuth" + BearerTokenAuth AuthType = "BearerToken" + HeadersAuth AuthType = "Header" + CookiesAuth AuthType = "Cookie" + QueryAuth AuthType = "Query" +) + +// SupportedAuthTypes returns the supported auth types +func SupportedAuthTypes() []string { + return []string{ + string(BasicAuth), + string(BearerTokenAuth), + string(HeadersAuth), + string(CookiesAuth), + string(QueryAuth), + } +} + +// Authx is a struct for secrets or credentials file +type Authx struct { + ID string `json:"id" yaml:"id"` + Info AuthFileInfo `json:"info" yaml:"info"` + Secrets []Secret `json:"static" yaml:"static"` +} + +type AuthFileInfo struct { + Name string `json:"name" yaml:"name"` + Author string `json:"author" yaml:"author"` + Severity string `json:"severity" yaml:"severity"` + Description string `json:"description" yaml:"description"` +} + +// Secret is a struct for secret or credential +type Secret struct { + Type string `json:"type" yaml:"type"` + Domains []string `json:"domains" yaml:"domains"` + DomainsRegex []string `json:"domains-regex" yaml:"domains-regex"` + Headers []KV `json:"headers" yaml:"headers"` // Headers preserve exact casing (useful for case-sensitive APIs) + Cookies []Cookie `json:"cookies" yaml:"cookies"` + Params []KV `json:"params" yaml:"params"` + Username string `json:"username" yaml:"username"` // can be either email or username + Password string `json:"password" yaml:"password"` + Token string `json:"token" yaml:"token"` // Bearer Auth token +} + +// GetStrategy returns the auth strategy for the secret +func (s *Secret) GetStrategy() AuthStrategy { + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + return NewBasicAuthStrategy(s) + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + return NewBearerTokenAuthStrategy(s) + case strings.EqualFold(s.Type, string(HeadersAuth)): + return NewHeadersAuthStrategy(s) + case strings.EqualFold(s.Type, string(CookiesAuth)): + return NewCookiesAuthStrategy(s) + case strings.EqualFold(s.Type, string(QueryAuth)): + return NewQueryAuthStrategy(s) + } + return nil +} + +func (s *Secret) Validate() error { + if !stringsutil.EqualFoldAny(s.Type, SupportedAuthTypes()...) { + return fmt.Errorf("invalid type: %s", s.Type) + } + if len(s.Domains) == 0 && len(s.DomainsRegex) == 0 { + return fmt.Errorf("domains or domains-regex cannot be empty") + } + if len(s.DomainsRegex) > 0 { + for _, domain := range s.DomainsRegex { + _, err := regexp.Compile(domain) + if err != nil { + return fmt.Errorf("invalid domain regex: %s", domain) + } + } + } + + switch { + case strings.EqualFold(s.Type, string(BasicAuth)): + if s.Username == "" { + return fmt.Errorf("username cannot be empty in basic auth") + } + if s.Password == "" { + return fmt.Errorf("password cannot be empty in basic auth") + } + case strings.EqualFold(s.Type, string(BearerTokenAuth)): + if s.Token == "" { + return fmt.Errorf("token cannot be empty in bearer token auth") + } + case strings.EqualFold(s.Type, string(HeadersAuth)): + if len(s.Headers) == 0 { + return fmt.Errorf("headers cannot be empty in headers auth") + } + for _, header := range s.Headers { + if err := header.Validate(); err != nil { + return fmt.Errorf("invalid header in headersAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(CookiesAuth)): + if len(s.Cookies) == 0 { + return fmt.Errorf("cookies cannot be empty in cookies auth") + } + for _, cookie := range s.Cookies { + if cookie.Raw != "" { + if err := cookie.Parse(); err != nil { + return fmt.Errorf("invalid raw cookie in cookiesAuth: %s", err) + } + } + if err := cookie.Validate(); err != nil { + return fmt.Errorf("invalid cookie in cookiesAuth: %s", err) + } + } + case strings.EqualFold(s.Type, string(QueryAuth)): + if len(s.Params) == 0 { + return fmt.Errorf("query cannot be empty in query auth") + } + for _, query := range s.Params { + if err := query.Validate(); err != nil { + return fmt.Errorf("invalid query in queryAuth: %s", err) + } + } + default: + return fmt.Errorf("invalid type: %s", s.Type) + } + return nil +} + +type KV struct { + Key string `json:"key" yaml:"key"` // Header key (preserves exact casing) + Value string `json:"value" yaml:"value"` +} + +func (k *KV) Validate() error { + if k.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if k.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +type Cookie struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + Raw string `json:"raw" yaml:"raw"` +} + +func (c *Cookie) Validate() error { + if c.Raw != "" { + return nil + } + if c.Key == "" { + return fmt.Errorf("key cannot be empty") + } + if c.Value == "" { + return fmt.Errorf("value cannot be empty") + } + return nil +} + +// Parse parses the cookie +// in raw the cookie is in format of +// Set-Cookie: =; Expires=; Path=; Domain=; Secure; HttpOnly +func (c *Cookie) Parse() error { + if c.Raw == "" { + return fmt.Errorf("raw cookie cannot be empty") + } + tmp := strings.TrimPrefix(c.Raw, "Set-Cookie: ") + slice := strings.Split(tmp, ";") + if len(slice) == 0 { + return fmt.Errorf("invalid raw cookie no ; found") + } + // first element is the cookie name and value + cookie := strings.Split(slice[0], "=") + if len(cookie) == 2 { + c.Key = cookie[0] + c.Value = cookie[1] + return nil + } + return fmt.Errorf("invalid raw cookie: %s", c.Raw) +} + +// GetAuthDataFromFile reads the auth data from file +func GetAuthDataFromFile(file string) (*Authx, error) { + ext := filepath.Ext(file) + if !generic.EqualsAny(ext, ".yml", ".yaml", ".json") { + return nil, fmt.Errorf("invalid file extension: supported extensions are .yml,.yaml and .json got %s", ext) + } + bin, err := os.ReadFile(file) + if err != nil { + return nil, err + } + if ext == ".yml" || ext == ".yaml" { + return GetAuthDataFromYAML(bin) + } + return GetAuthDataFromJSON(bin) +} + +// GetAuthDataFromYAML reads the auth data from yaml +func GetAuthDataFromYAML(data []byte) (*Authx, error) { + var auth Authx + err := yaml.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal yaml") + return nil, errorErr + } + return &auth, nil +} + +// GetAuthDataFromJSON reads the auth data from json +func GetAuthDataFromJSON(data []byte) (*Authx, error) { + var auth Authx + err := json.Unmarshal(data, &auth) + if err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("could not unmarshal json") + return nil, errorErr + } + return &auth, nil +} diff --git a/common/authprovider/authx/headers_auth.go b/common/authprovider/authx/headers_auth.go new file mode 100644 index 0000000..d474f75 --- /dev/null +++ b/common/authprovider/authx/headers_auth.go @@ -0,0 +1,39 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +var ( + _ AuthStrategy = &HeadersAuthStrategy{} +) + +// HeadersAuthStrategy is a strategy for headers auth +type HeadersAuthStrategy struct { + Data *Secret +} + +// NewHeadersAuthStrategy creates a new headers auth strategy +func NewHeadersAuthStrategy(data *Secret) *HeadersAuthStrategy { + return &HeadersAuthStrategy{Data: data} +} + +// Apply applies the headers auth strategy to the request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) Apply(req *http.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} + +// ApplyOnRR applies the headers auth strategy to the retryable request +// NOTE: This preserves exact header casing (e.g., barAuthToken stays as barAuthToken) +// This is useful for APIs that require case-sensitive header names +func (s *HeadersAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + for _, header := range s.Data.Headers { + req.Header[header.Key] = []string{header.Value} + } +} diff --git a/common/authprovider/authx/query_auth.go b/common/authprovider/authx/query_auth.go new file mode 100644 index 0000000..796d8b1 --- /dev/null +++ b/common/authprovider/authx/query_auth.go @@ -0,0 +1,42 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + _ AuthStrategy = &QueryAuthStrategy{} +) + +// QueryAuthStrategy is a strategy for query auth +type QueryAuthStrategy struct { + Data *Secret +} + +// NewQueryAuthStrategy creates a new query auth strategy +func NewQueryAuthStrategy(data *Secret) *QueryAuthStrategy { + return &QueryAuthStrategy{Data: data} +} + +// Apply applies the query auth strategy to the request +func (s *QueryAuthStrategy) Apply(req *http.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.URL.RawQuery = q.Encode() +} + +// ApplyOnRR applies the query auth strategy to the retryable request +func (s *QueryAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + q := urlutil.NewOrderedParams() + q.Decode(req.Request.URL.RawQuery) + for _, p := range s.Data.Params { + q.Add(p.Key, p.Value) + } + req.Request.URL.RawQuery = q.Encode() +} diff --git a/common/authprovider/authx/strategy.go b/common/authprovider/authx/strategy.go new file mode 100644 index 0000000..35e42e3 --- /dev/null +++ b/common/authprovider/authx/strategy.go @@ -0,0 +1,16 @@ +package authx + +import ( + "net/http" + + "github.com/projectdiscovery/retryablehttp-go" +) + +// AuthStrategy is an interface for auth strategies +// basic auth , bearer token, headers, cookies, query +type AuthStrategy interface { + // Apply applies the strategy to the request + Apply(*http.Request) + // ApplyOnRR applies the strategy to the retryable request + ApplyOnRR(*retryablehttp.Request) +} diff --git a/common/authprovider/file.go b/common/authprovider/file.go new file mode 100644 index 0000000..465cd06 --- /dev/null +++ b/common/authprovider/file.go @@ -0,0 +1,114 @@ +package authprovider + +import ( + "net" + "net/url" + "regexp" + "strings" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + "github.com/projectdiscovery/utils/errkit" + urlutil "github.com/projectdiscovery/utils/url" +) + +// FileAuthProvider is an auth provider for file based auth +// it accepts a secrets file and returns its provider +type FileAuthProvider struct { + Path string + store *authx.Authx + compiled map[*regexp.Regexp][]authx.AuthStrategy + domains map[string][]authx.AuthStrategy +} + +// NewFileAuthProvider creates a new file based auth provider +func NewFileAuthProvider(path string) (AuthProvider, error) { + store, err := authx.GetAuthDataFromFile(path) + if err != nil { + return nil, err + } + if len(store.Secrets) == 0 { + return nil, ErrNoSecrets + } + for _, secret := range store.Secrets { + if err := secret.Validate(); err != nil { + errorErr := errkit.FromError(err) + errorErr.Msgf("invalid secret in file: %s", path) + return nil, errorErr + } + } + f := &FileAuthProvider{Path: path, store: store} + f.init() + return f, nil +} + +// init initializes the file auth provider +func (f *FileAuthProvider) init() { + for _, _secret := range f.store.Secrets { + secret := _secret // allocate copy of pointer + if len(secret.DomainsRegex) > 0 { + for _, domain := range secret.DomainsRegex { + if f.compiled == nil { + f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy) + } + compiled, err := regexp.Compile(domain) + if err != nil { + continue + } + + if ss, ok := f.compiled[compiled]; ok { + f.compiled[compiled] = append(ss, secret.GetStrategy()) + } else { + f.compiled[compiled] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } + for _, domain := range secret.Domains { + if f.domains == nil { + f.domains = make(map[string][]authx.AuthStrategy) + } + domain = strings.TrimSpace(domain) + domain = strings.TrimSuffix(domain, ":80") + domain = strings.TrimSuffix(domain, ":443") + if ss, ok := f.domains[domain]; ok { + f.domains[domain] = append(ss, secret.GetStrategy()) + } else { + f.domains[domain] = []authx.AuthStrategy{secret.GetStrategy()} + } + } + } +} + +// LookupAddr looks up a given domain/address and returns appropriate auth strategy +func (f *FileAuthProvider) LookupAddr(addr string) []authx.AuthStrategy { + var strategies []authx.AuthStrategy + + if strings.Contains(addr, ":") { + // default normalization for host:port + host, port, err := net.SplitHostPort(addr) + if err == nil && (port == "80" || port == "443") { + addr = host + } + } + for domain, strategy := range f.domains { + if strings.EqualFold(domain, addr) { + strategies = append(strategies, strategy...) + } + } + for compiled, strategy := range f.compiled { + if compiled.MatchString(addr) { + strategies = append(strategies, strategy...) + } + } + + return strategies +} + +// LookupURL looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} + +// LookupURLX looks up a given URL and returns appropriate auth strategy +func (f *FileAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + return f.LookupAddr(u.Host) +} diff --git a/common/authprovider/interface.go b/common/authprovider/interface.go new file mode 100644 index 0000000..981496f --- /dev/null +++ b/common/authprovider/interface.go @@ -0,0 +1,53 @@ +// TODO: This package should be abstracted out to projectdiscovery/utils +// so it can be shared between httpx, nuclei, and other tools. +package authprovider + +import ( + "fmt" + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +var ( + ErrNoSecrets = fmt.Errorf("no secrets in given provider") +) + +var ( + _ AuthProvider = &FileAuthProvider{} +) + +// AuthProvider is an interface for auth providers +// It implements a data structure suitable for quick lookup and retrieval +// of auth strategies +type AuthProvider interface { + // LookupAddr looks up a given domain/address and returns appropriate auth strategy + // for it (accepted inputs are scanme.sh or scanme.sh:443) + LookupAddr(string) []authx.AuthStrategy + // LookupURL looks up a given URL and returns appropriate auth strategy + // it accepts a valid url struct and returns the auth strategy + LookupURL(*url.URL) []authx.AuthStrategy + // LookupURLX looks up a given URL and returns appropriate auth strategy + // it accepts pd url struct (i.e urlutil.URL) and returns the auth strategy + LookupURLX(*urlutil.URL) []authx.AuthStrategy +} + +// AuthProviderOptions contains options for the auth provider +type AuthProviderOptions struct { + // File based auth provider options + SecretsFiles []string +} + +// NewAuthProvider creates a new auth provider from the given options +func NewAuthProvider(options *AuthProviderOptions) (AuthProvider, error) { + var providers []AuthProvider + for _, file := range options.SecretsFiles { + provider, err := NewFileAuthProvider(file) + if err != nil { + return nil, err + } + providers = append(providers, provider) + } + return NewMultiAuthProvider(providers...), nil +} diff --git a/common/authprovider/multi.go b/common/authprovider/multi.go new file mode 100644 index 0000000..24d59f0 --- /dev/null +++ b/common/authprovider/multi.go @@ -0,0 +1,50 @@ +package authprovider + +import ( + "net/url" + + "github.com/projectdiscovery/httpx/common/authprovider/authx" + urlutil "github.com/projectdiscovery/utils/url" +) + +// MultiAuthProvider is a convenience wrapper for multiple auth providers +// it returns the first matching auth strategy for a given domain +// if there are multiple auth strategies for a given domain, it returns the first one +type MultiAuthProvider struct { + Providers []AuthProvider +} + +// NewMultiAuthProvider creates a new multi auth provider +func NewMultiAuthProvider(providers ...AuthProvider) AuthProvider { + return &MultiAuthProvider{Providers: providers} +} + +func (m *MultiAuthProvider) LookupAddr(host string) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupAddr(host) + if len(strategy) > 0 { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURL(u *url.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURL(u) + if strategy != nil { + return strategy + } + } + return nil +} + +func (m *MultiAuthProvider) LookupURLX(u *urlutil.URL) []authx.AuthStrategy { + for _, provider := range m.Providers { + strategy := provider.LookupURLX(u) + if strategy != nil { + return strategy + } + } + return nil +} diff --git a/runner/options.go b/runner/options.go index 46e9bc8..69c3f8b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -351,6 +351,8 @@ type Options struct { // AssetFileUpload AssetFileUpload string TeamID string + // SecretFile is the path to the secret file for authentication + SecretFile string // OnClose adds a callback function that is invoked when httpx is closed // to be exact at end of existing closures OnClose func() @@ -523,6 +525,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "Disable Stdin processing"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), + flagSet.StringVarP(&options.SecretFile, "secret-file", "sf", "", "path to the secret file for authentication"), ) flagSet.CreateGroup("debug", "Debug", @@ -677,6 +680,10 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } + if options.SecretFile != "" && !fileutil.FileExists(options.SecretFile) { + return fmt.Errorf("secret file '%s' does not exist", options.SecretFile) + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 2a20265..0d0ad1e 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -36,6 +36,7 @@ import ( "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/httpx/common/pagetypeclassifier" + "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/static" "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/networkpolicy" @@ -94,6 +95,7 @@ type Runner struct { pHashClusters []pHashCluster simHashes gcache.Cache[uint64, struct{}] // Include simHashes for efficient duplicate detection httpApiEndpoint *Server + authProvider authprovider.AuthProvider } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -412,6 +414,16 @@ func New(options *Options) (*Runner, error) { } runner.pageTypeClassifier = pageTypeClassifier + if options.SecretFile != "" { + authProviderOpts := &authprovider.AuthProviderOptions{ + SecretsFiles: []string{options.SecretFile}, + } + runner.authProvider, err = authprovider.NewAuthProvider(authProviderOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create auth provider") + } + } + if options.HttpApiEndpoint != "" { apiServer := NewServer(options.HttpApiEndpoint, options) gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint) @@ -1705,6 +1717,16 @@ retry: } hp.SetCustomHeaders(req, hp.CustomHeaders) + + // Apply auth strategies if auth provider is configured + if r.authProvider != nil { + if strategies := r.authProvider.LookupURLX(URL); len(strategies) > 0 { + for _, strategy := range strategies { + strategy.ApplyOnRR(req) + } + } + } + // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody))