From 978addb8bd190e3c9d5a526e4bf619fd1b6c9eb1 Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:02:44 +0530 Subject: [PATCH 1/4] feat: add URLScan.io as passive subdomain source Add URLScan.io as a new passive subdomain enumeration source with full pagination support and robust rate limiting handling. Features: - Fetches subdomains from URLScan.io Search API - Implements cursor-based pagination using search_after parameter - Extracts domains from task.domain, task.url, page.domain, page.url fields - Requires API key (free tier available at urlscan.io) Rate Limiting: - Conservative pagination delay (10s between pages) to respect strict burst limits - Exponential backoff retry logic for 429/503 responses - Respects X-Rate-Limit-Reset-After header for dynamic backoff - Limited to 5 pages max (500 results) to avoid quota exhaustion Configuration: - Max 5 pages per enumeration (configurable via maxPages constant) - 100 results per page (configurable via maxPerPage constant) - 2 retry attempts for rate-limited requests - 20 second initial backoff, doubles on each retry Changes: - pkg/subscraping/sources/urlscan/urlscan.go: New URLScan source implementation - pkg/passive/sources.go: Register URLScan source - pkg/passive/sources_test.go: Add URLScan to test lists - pkg/runner/options.go: Add urlscan to source options - .github/workflows/build-test.yml: Add URLSCAN_API_KEY secret Closes: Feature request for URLScan.io integration --- .github/workflows/build-test.yml | 1 + pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 3 + pkg/runner/options.go | 1 + pkg/subscraping/sources/urlscan/urlscan.go | 295 +++++++++++++++++++++ 5 files changed, 302 insertions(+) create mode 100644 pkg/subscraping/sources/urlscan/urlscan.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d567636b..c6728ea5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -65,6 +65,7 @@ jobs: SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 97877f85..c5d3a514 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -53,6 +53,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" @@ -114,6 +115,7 @@ var AllSources = [...]subscraping.Source{ &hudsonrock.Source{}, &digitalyama.Source{}, &thc.Source{}, + &urlscan.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 82aa0336..7c4c0da3 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -63,6 +63,7 @@ var ( "digitalyama", "merklemap", "thc", + "urlscan", } expectedDefaultSources = []string{ @@ -105,6 +106,7 @@ var ( "builtwith", "digitalyama", "thc", + "urlscan", } expectedDefaultRecursiveSources = []string{ @@ -121,6 +123,7 @@ var ( "leakix", "facebook", "merklemap", + "urlscan", // "reconcloud", } ) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index d3284cab..946c3027 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -265,4 +265,5 @@ var defaultRateLimits = []string{ // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", + "urlscan=1/s", } diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go new file mode 100644 index 00000000..584acbbd --- /dev/null +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -0,0 +1,295 @@ +// Package urlscan logic +package urlscan + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the URLScan API base URL + baseURL = "https://urlscan.io/api/v1/search/" + // maxPages is the maximum number of pages to fetch (reduced due to strict rate limits) + // URLScan has very strict burst limiting, so we limit to 3-5 pages max + maxPages = 5 + // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) + maxPerPage = 100 + // maxRetries is the number of retry attempts for rate-limited requests + // Reducing retries since each retry consumes quota + maxRetries = 2 + // initialBackoff is the initial wait time before retrying (URLScan recommends waiting) + initialBackoff = 20 * time.Second + // paginationDelay is the delay between pagination requests to avoid rate limits + // URLScan has 120 requests/minute but very strict burst limits + // Using 8-10 seconds to be extra conservative + paginationDelay = 10 * time.Second +) + +// response represents the URLScan API response structure +type response struct { + Results []struct { + Task struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"task"` + Page struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"page"` + Sort []interface{} `json:"sort"` + } `json:"results"` + HasMore bool `json:"has_more"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + s.skipped = false + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"api-key": randomApiKey} + + // Search with wildcard to get more subdomain results + s.enumerate(ctx, domain, headers, session, results) + }() + + return results +} + +// enumerate performs the actual enumeration with pagination +func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + var searchAfter string + currentPage := 0 + + for { + // Check context at the start of each iteration (standard pattern) + select { + case <-ctx.Done(): + return + default: + } + + // Check max pages limit (similar to Censys) + if currentPage >= maxPages { + break + } + + // Build search URL - search for domain and all subdomains + searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) + if searchAfter != "" { + searchURL += "&search_after=" + url.QueryEscape(searchAfter) + } + + resp, err := s.makeRequestWithRetry(ctx, session, searchURL, headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + // Process results - extract subdomains from multiple fields + for _, result := range data.Results { + // Extract from task.domain, task.url, page.domain, page.url + candidates := []string{ + result.Task.Domain, + result.Page.Domain, + } + + // Also extract from URLs if present + if result.Task.URL != "" { + if u, err := url.Parse(result.Task.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + if result.Page.URL != "" { + if u, err := url.Parse(result.Page.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, subdomain := range session.Extractor.Extract(candidate) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + } + + // Check pagination conditions (similar to Shodan/VirusTotal pattern) + if !data.HasMore || len(data.Results) == 0 { + break + } + + // Get sort value for next page + lastResult := data.Results[len(data.Results)-1] + if len(lastResult.Sort) == 0 { + break + } + + // Build search_after parameter + sortValues := make([]string, len(lastResult.Sort)) + for i, v := range lastResult.Sort { + switch val := v.(type) { + case float64: + sortValues[i] = fmt.Sprintf("%.0f", val) + default: + sortValues[i] = fmt.Sprintf("%v", v) + } + } + // Don't URL encode here - the session.Get will handle encoding + searchAfter = strings.Join(sortValues, ",") + currentPage++ + + // Delay between pages to respect rate limits + select { + case <-ctx.Done(): + return + case <-time.After(paginationDelay): + } + } +} + +// makeRequestWithRetry handles rate limiting (429) and server errors (503) with exponential backoff +// This is necessary for URLScan as it has stricter rate limits than other sources +func (s *Source) makeRequestWithRetry(ctx context.Context, session *subscraping.Session, searchURL string, headers map[string]string) (*http.Response, error) { + var resp *http.Response + var err error + backoff := initialBackoff + + for attempt := 0; attempt <= maxRetries; attempt++ { + s.requests++ + resp, err = session.Get(ctx, searchURL, "", headers) + + // If request succeeded, return it + if err == nil && resp != nil && resp.StatusCode == http.StatusOK { + return resp, nil + } + + // Check for retryable status codes (429 rate limited, 503 server overload) + if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) { + // Try to get recommended wait time from X-Rate-Limit-Reset-After header + resetAfter := resp.Header.Get("X-Rate-Limit-Reset-After") + if resetAfter != "" { + // Parse the seconds and use it as wait time + if seconds, parseErr := time.ParseDuration(resetAfter + "s"); parseErr == nil && seconds > 0 { + backoff = seconds + (2 * time.Second) // Add 2 extra seconds as buffer + } + } + + session.DiscardHTTPResponse(resp) + + if attempt < maxRetries { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff): + backoff *= 2 // Exponential backoff for next retry + } + continue + } + return nil, fmt.Errorf("rate limited (status %d) after %d retries", resp.StatusCode, maxRetries+1) + } + + // For other errors (non-200, non-retryable status codes), return an error + // This prevents callers from trying to decode error response bodies + if resp != nil { + status := resp.StatusCode + session.DiscardHTTPResponse(resp) + if err == nil { + err = fmt.Errorf("unexpected status %d", status) + } + } else if err == nil { + err = fmt.Errorf("unexpected nil response") + } + return nil, err + } + + // This return is required by the Go compiler but is technically unreachable + // since all paths in the loop above return + return nil, fmt.Errorf("max retries exceeded") +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "urlscan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From e8c0e16d6db51a6f2fe7fa82b0dd23864cbc6f9b Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:42:20 +0530 Subject: [PATCH 2/4] fix: replace deprecated errorutil with fmt.Errorf in facebook source Replace deprecated github.com/projectdiscovery/utils/errors package with standard Go error wrapping using fmt.Errorf to fix staticcheck SA1019 linter errors. --- pkg/subscraping/sources/facebook/ctlogs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index 444db88e..dab71b8d 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -10,7 +10,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - errorutil "github.com/projectdiscovery/utils/errors" "github.com/projectdiscovery/utils/generic" urlutil "github.com/projectdiscovery/utils/url" ) @@ -128,7 +127,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)} return } for _, v := range response.Data { From 675be65d479a736cf2fe70b61f9d19b74f774a6e Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 17:57:10 +0530 Subject: [PATCH 3/4] test: skip flaky sources in TestSourcesWithoutKeys Add leakix, reconeer, and sitedossier to ignored sources list: - leakix: now requires API key (returns 401) - reconeer: now requires API key (returns 401) - sitedossier: flaky, returns no results in CI --- pkg/passive/sources_wo_auth_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go index 01fc57d9..c3316d63 100644 --- a/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -34,6 +34,9 @@ func TestSourcesWithoutKeys(t *testing.T) { "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" + "leakix", // now requires API key (returns 401) + "reconeer", // now requires API key (returns 401) + "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" From 58c1e5ac829cc0aa4cc504ad795c2f4842f0348f Mon Sep 17 00:00:00 2001 From: Jigardjain Date: Fri, 30 Jan 2026 18:05:47 +0530 Subject: [PATCH 4/4] refactor: simplify urlscan source to use session rate limiting Remove custom pagination delay and retry logic since the session already handles rate limiting via MultiRateLimiter. This aligns with how other sources (shodan, virustotal) are implemented. --- pkg/subscraping/sources/urlscan/urlscan.go | 92 ++-------------------- 1 file changed, 6 insertions(+), 86 deletions(-) diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go index 584acbbd..1b0a9fcf 100644 --- a/pkg/subscraping/sources/urlscan/urlscan.go +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -4,7 +4,6 @@ package urlscan import ( "context" "fmt" - "net/http" "net/url" "strings" "time" @@ -17,20 +16,10 @@ import ( const ( // baseURL is the URLScan API base URL baseURL = "https://urlscan.io/api/v1/search/" - // maxPages is the maximum number of pages to fetch (reduced due to strict rate limits) - // URLScan has very strict burst limiting, so we limit to 3-5 pages max + // maxPages is the maximum number of pages to fetch maxPages = 5 // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) maxPerPage = 100 - // maxRetries is the number of retry attempts for rate-limited requests - // Reducing retries since each retry consumes quota - maxRetries = 2 - // initialBackoff is the initial wait time before retrying (URLScan recommends waiting) - initialBackoff = 20 * time.Second - // paginationDelay is the delay between pagination requests to avoid rate limits - // URLScan has 120 requests/minute but very strict burst limits - // Using 8-10 seconds to be extra conservative - paginationDelay = 10 * time.Second ) // response represents the URLScan API response structure @@ -95,28 +84,28 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin currentPage := 0 for { - // Check context at the start of each iteration (standard pattern) select { case <-ctx.Done(): return default: } - // Check max pages limit (similar to Censys) if currentPage >= maxPages { break } - // Build search URL - search for domain and all subdomains + // Build search URL searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) if searchAfter != "" { searchURL += "&search_after=" + url.QueryEscape(searchAfter) } - resp, err := s.makeRequestWithRetry(ctx, session, searchURL, headers) + s.requests++ + resp, err := session.Get(ctx, searchURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ + session.DiscardHTTPResponse(resp) return } @@ -132,7 +121,6 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin // Process results - extract subdomains from multiple fields for _, result := range data.Results { - // Extract from task.domain, task.url, page.domain, page.url candidates := []string{ result.Task.Domain, result.Page.Domain, @@ -165,7 +153,7 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin } } - // Check pagination conditions (similar to Shodan/VirusTotal pattern) + // Check pagination conditions if !data.HasMore || len(data.Results) == 0 { break } @@ -186,79 +174,11 @@ func (s *Source) enumerate(ctx context.Context, domain string, headers map[strin sortValues[i] = fmt.Sprintf("%v", v) } } - // Don't URL encode here - the session.Get will handle encoding searchAfter = strings.Join(sortValues, ",") currentPage++ - - // Delay between pages to respect rate limits - select { - case <-ctx.Done(): - return - case <-time.After(paginationDelay): - } } } -// makeRequestWithRetry handles rate limiting (429) and server errors (503) with exponential backoff -// This is necessary for URLScan as it has stricter rate limits than other sources -func (s *Source) makeRequestWithRetry(ctx context.Context, session *subscraping.Session, searchURL string, headers map[string]string) (*http.Response, error) { - var resp *http.Response - var err error - backoff := initialBackoff - - for attempt := 0; attempt <= maxRetries; attempt++ { - s.requests++ - resp, err = session.Get(ctx, searchURL, "", headers) - - // If request succeeded, return it - if err == nil && resp != nil && resp.StatusCode == http.StatusOK { - return resp, nil - } - - // Check for retryable status codes (429 rate limited, 503 server overload) - if resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable) { - // Try to get recommended wait time from X-Rate-Limit-Reset-After header - resetAfter := resp.Header.Get("X-Rate-Limit-Reset-After") - if resetAfter != "" { - // Parse the seconds and use it as wait time - if seconds, parseErr := time.ParseDuration(resetAfter + "s"); parseErr == nil && seconds > 0 { - backoff = seconds + (2 * time.Second) // Add 2 extra seconds as buffer - } - } - - session.DiscardHTTPResponse(resp) - - if attempt < maxRetries { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(backoff): - backoff *= 2 // Exponential backoff for next retry - } - continue - } - return nil, fmt.Errorf("rate limited (status %d) after %d retries", resp.StatusCode, maxRetries+1) - } - - // For other errors (non-200, non-retryable status codes), return an error - // This prevents callers from trying to decode error response bodies - if resp != nil { - status := resp.StatusCode - session.DiscardHTTPResponse(resp) - if err == nil { - err = fmt.Errorf("unexpected status %d", status) - } - } else if err == nil { - err = fmt.Errorf("unexpected nil response") - } - return nil, err - } - - // This return is required by the Go compiler but is technically unreachable - // since all paths in the loop above return - return nil, fmt.Errorf("max retries exceeded") -} - // Name returns the name of the source func (s *Source) Name() string { return "urlscan"