mirror of
https://github.com/projectdiscovery/subfinder
synced 2026-06-21 14:05:24 +00:00
Charles Wong
c1ce78eac9
Two bugs in rate limit handling: 1. Global -rl flag was ignored for sources without a per-source override. Only sources explicitly listed in -rls got rate-limited; all others ran unlimited, defeating the purpose of -rl. 2. Per-source duration from -rls (e.g. hackertarget=2/m) was silently dropped. runner.go stored MaxCount but discarded Duration, so all sources defaulted to per-second regardless of the /m or /s suffix. Root cause: runner.go converted goflags.RateLimitMap → CustomRateLimit but only copied MaxCount. buildMultiRateLimiter only checked per-source limits, never falling back to the global rate. Changes: - Add CustomDuration field to CustomRateLimit struct - Store both MaxCount and Duration from -rls in runner.go - Add resolveSourceRateLimit() with clear priority: per-source > global > unlimited - Guard against nil rateLimit in buildMultiRateLimiter - Add 9 unit tests covering all combinations Fixes #1434
113 lines
2.6 KiB
Go
113 lines
2.6 KiB
Go
package subscraping
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/projectdiscovery/ratelimit"
|
|
mapsutil "github.com/projectdiscovery/utils/maps"
|
|
)
|
|
|
|
type CtxArg string
|
|
|
|
const (
|
|
CtxSourceArg CtxArg = "source"
|
|
)
|
|
|
|
type CustomRateLimit struct {
|
|
Custom mapsutil.SyncLockMap[string, uint]
|
|
CustomDuration mapsutil.SyncLockMap[string, time.Duration]
|
|
}
|
|
|
|
// BasicAuth request's Authorization header
|
|
type BasicAuth struct {
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
// Statistics contains statistics about the scraping process
|
|
type Statistics struct {
|
|
TimeTaken time.Duration
|
|
Requests int
|
|
Errors int
|
|
Results int
|
|
Skipped bool
|
|
}
|
|
|
|
// KeyRequirement represents the API key requirement level for a source
|
|
type KeyRequirement int
|
|
|
|
const (
|
|
NoKey KeyRequirement = iota
|
|
OptionalKey
|
|
RequiredKey
|
|
)
|
|
|
|
// Source is an interface inherited by each passive source
|
|
type Source interface {
|
|
// Run takes a domain as argument and a session object
|
|
// which contains the extractor for subdomains, http client
|
|
// and other stuff.
|
|
Run(context.Context, string, *Session) <-chan Result
|
|
|
|
// Name returns the name of the source. It is preferred to use lower case names.
|
|
Name() string
|
|
|
|
// IsDefault returns true if the current source should be
|
|
// used as part of the default execution.
|
|
IsDefault() bool
|
|
|
|
// HasRecursiveSupport returns true if the current source
|
|
// accepts subdomains (e.g. subdomain.domain.tld),
|
|
// not just root domains.
|
|
HasRecursiveSupport() bool
|
|
|
|
// KeyRequirement returns the API key requirement level for this source
|
|
KeyRequirement() KeyRequirement
|
|
|
|
// NeedsKey returns true if the source requires an API key.
|
|
// Deprecated: Use KeyRequirement() instead for more granular control.
|
|
NeedsKey() bool
|
|
|
|
AddApiKeys([]string)
|
|
|
|
// Statistics returns the scrapping statistics for the source
|
|
Statistics() Statistics
|
|
}
|
|
|
|
// SubdomainExtractor is an interface that defines the contract for subdomain extraction.
|
|
type SubdomainExtractor interface {
|
|
Extract(text string) []string
|
|
}
|
|
|
|
// Session is the option passed to the source, an option is created
|
|
// uniquely for each source.
|
|
type Session struct {
|
|
//SubdomainExtractor
|
|
Extractor SubdomainExtractor
|
|
// Client is the current http client
|
|
Client *http.Client
|
|
// Rate limit instance
|
|
MultiRateLimiter *ratelimit.MultiLimiter
|
|
// Timeout is the timeout in seconds for requests
|
|
Timeout int
|
|
}
|
|
|
|
// Result is a result structure returned by a source
|
|
type Result struct {
|
|
Type ResultType
|
|
Source string
|
|
Value string
|
|
Error error
|
|
}
|
|
|
|
// ResultType is the type of result returned by the source
|
|
type ResultType int
|
|
|
|
// Types of results returned by the source
|
|
const (
|
|
Subdomain ResultType = iota
|
|
Error
|
|
)
|