diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index d567636b..c6728ea5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -65,6 +65,7 @@ jobs: SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 97877f85..c5d3a514 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -53,6 +53,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" @@ -114,6 +115,7 @@ var AllSources = [...]subscraping.Source{ &hudsonrock.Source{}, &digitalyama.Source{}, &thc.Source{}, + &urlscan.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 82aa0336..7c4c0da3 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -63,6 +63,7 @@ var ( "digitalyama", "merklemap", "thc", + "urlscan", } expectedDefaultSources = []string{ @@ -105,6 +106,7 @@ var ( "builtwith", "digitalyama", "thc", + "urlscan", } expectedDefaultRecursiveSources = []string{ @@ -121,6 +123,7 @@ var ( "leakix", "facebook", "merklemap", + "urlscan", // "reconcloud", } ) diff --git a/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go index 01fc57d9..c3316d63 100644 --- a/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -34,6 +34,9 @@ func TestSourcesWithoutKeys(t *testing.T) { "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" + "leakix", // now requires API key (returns 401) + "reconeer", // now requires API key (returns 401) + "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" diff --git a/pkg/runner/options.go b/pkg/runner/options.go index d3284cab..946c3027 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -265,4 +265,5 @@ var defaultRateLimits = []string{ // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", + "urlscan=1/s", } diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index 444db88e..dab71b8d 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -10,7 +10,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - errorutil "github.com/projectdiscovery/utils/errors" "github.com/projectdiscovery/utils/generic" urlutil "github.com/projectdiscovery/utils/url" ) @@ -128,7 +127,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)} return } for _, v := range response.Data { diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go new file mode 100644 index 00000000..1b0a9fcf --- /dev/null +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -0,0 +1,215 @@ +// Package urlscan logic +package urlscan + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the URLScan API base URL + baseURL = "https://urlscan.io/api/v1/search/" + // maxPages is the maximum number of pages to fetch + maxPages = 5 + // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) + maxPerPage = 100 +) + +// response represents the URLScan API response structure +type response struct { + Results []struct { + Task struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"task"` + Page struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"page"` + Sort []interface{} `json:"sort"` + } `json:"results"` + HasMore bool `json:"has_more"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + s.skipped = false + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"api-key": randomApiKey} + + // Search with wildcard to get more subdomain results + s.enumerate(ctx, domain, headers, session, results) + }() + + return results +} + +// enumerate performs the actual enumeration with pagination +func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + var searchAfter string + currentPage := 0 + + for { + select { + case <-ctx.Done(): + return + default: + } + + if currentPage >= maxPages { + break + } + + // Build search URL + searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) + if searchAfter != "" { + searchURL += "&search_after=" + url.QueryEscape(searchAfter) + } + + s.requests++ + resp, err := session.Get(ctx, searchURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + // Process results - extract subdomains from multiple fields + for _, result := range data.Results { + candidates := []string{ + result.Task.Domain, + result.Page.Domain, + } + + // Also extract from URLs if present + if result.Task.URL != "" { + if u, err := url.Parse(result.Task.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + if result.Page.URL != "" { + if u, err := url.Parse(result.Page.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, subdomain := range session.Extractor.Extract(candidate) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + } + + // Check pagination conditions + if !data.HasMore || len(data.Results) == 0 { + break + } + + // Get sort value for next page + lastResult := data.Results[len(data.Results)-1] + if len(lastResult.Sort) == 0 { + break + } + + // Build search_after parameter + sortValues := make([]string, len(lastResult.Sort)) + for i, v := range lastResult.Sort { + switch val := v.(type) { + case float64: + sortValues[i] = fmt.Sprintf("%.0f", val) + default: + sortValues[i] = fmt.Sprintf("%v", v) + } + } + searchAfter = strings.Join(sortValues, ",") + currentPage++ + } +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "urlscan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +}