From e96320ec383b556de8ea6c7d3fb261ad2a4bc0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 5 Jan 2026 14:14:20 +0300 Subject: [PATCH] add request tracking to -stats flag Adds request counting to all sources to help users monitor API usage and debug quota consumption issues like #1562. Changes: - Add Requests field to Statistics struct - Track HTTP requests in all 53 sources - Display request count in stats output Closes #1698 --- pkg/runner/stats.go | 4 ++-- pkg/subscraping/sources/alienvault/alienvault.go | 4 ++++ pkg/subscraping/sources/anubis/anubis.go | 4 ++++ pkg/subscraping/sources/bevigil/bevigil.go | 4 ++++ pkg/subscraping/sources/bufferover/bufferover.go | 4 ++++ pkg/subscraping/sources/builtwith/builtwith.go | 4 ++++ pkg/subscraping/sources/c99/c99.go | 4 ++++ pkg/subscraping/sources/censys/censys.go | 4 ++++ pkg/subscraping/sources/certspotter/certspotter.go | 5 +++++ pkg/subscraping/sources/chaos/chaos.go | 4 ++++ pkg/subscraping/sources/chinaz/chinaz.go | 4 ++++ pkg/subscraping/sources/commoncrawl/commoncrawl.go | 5 +++++ pkg/subscraping/sources/crtsh/crtsh.go | 4 ++++ pkg/subscraping/sources/digitalyama/digitalyama.go | 4 ++++ pkg/subscraping/sources/digitorus/digitorus.go | 4 ++++ pkg/subscraping/sources/dnsdb/dnsdb.go | 5 +++++ pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 4 ++++ pkg/subscraping/sources/dnsrepo/dnsrepo.go | 4 ++++ pkg/subscraping/sources/domainsproject/domainsproject.go | 4 ++++ pkg/subscraping/sources/driftnet/driftnet.go | 4 ++++ pkg/subscraping/sources/facebook/ctlogs.go | 4 ++++ pkg/subscraping/sources/fofa/fofa.go | 4 ++++ pkg/subscraping/sources/fullhunt/fullhunt.go | 4 ++++ pkg/subscraping/sources/github/github.go | 5 +++++ pkg/subscraping/sources/gitlab/gitlab.go | 5 +++++ pkg/subscraping/sources/hackertarget/hackertarget.go | 4 ++++ pkg/subscraping/sources/hudsonrock/hudsonrock.go | 4 ++++ pkg/subscraping/sources/intelx/intelx.go | 5 +++++ pkg/subscraping/sources/leakix/leakix.go | 4 ++++ pkg/subscraping/sources/merklemap/merklemap.go | 4 ++++ pkg/subscraping/sources/netlas/netlas.go | 5 +++++ pkg/subscraping/sources/onyphe/onyphe.go | 4 ++++ pkg/subscraping/sources/profundis/profundis.go | 4 ++++ pkg/subscraping/sources/pugrecon/pugrecon.go | 4 ++++ pkg/subscraping/sources/quake/quake.go | 4 ++++ pkg/subscraping/sources/rapiddns/rapiddns.go | 4 ++++ pkg/subscraping/sources/reconcloud/reconcloud.go | 4 ++++ pkg/subscraping/sources/reconeer/reconeer.go | 4 ++++ pkg/subscraping/sources/redhuntlabs/redhuntlabs.go | 5 +++++ pkg/subscraping/sources/riddler/riddler.go | 4 ++++ pkg/subscraping/sources/rsecloud/rsecloud.go | 4 ++++ pkg/subscraping/sources/securitytrails/securitytrails.go | 6 ++++++ pkg/subscraping/sources/shodan/shodan.go | 4 ++++ pkg/subscraping/sources/sitedossier/sitedossier.go | 4 ++++ pkg/subscraping/sources/thc/thc.go | 4 ++++ pkg/subscraping/sources/threatbook/threatbook.go | 4 ++++ pkg/subscraping/sources/threatcrowd/threatcrowd.go | 4 ++++ pkg/subscraping/sources/threatminer/threatminer.go | 4 ++++ pkg/subscraping/sources/virustotal/virustotal.go | 4 ++++ pkg/subscraping/sources/waybackarchive/waybackarchive.go | 4 ++++ pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go | 4 ++++ pkg/subscraping/sources/windvane/windvane.go | 4 ++++ pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go | 4 ++++ pkg/subscraping/types.go | 1 + 54 files changed, 221 insertions(+), 2 deletions(-) diff --git a/pkg/runner/stats.go b/pkg/runner/stats.go index d34b20f8..ccaa13e3 100644 --- a/pkg/runner/stats.go +++ b/pkg/runner/stats.go @@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) { if sourceStats.Skipped { skipped = append(skipped, fmt.Sprintf(" %s", source)) } else { - lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors)) + lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors)) } } if len(lines) > 0 { - gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) + gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68)) gologger.Print().Msg(strings.Join(lines, "\n")) gologger.Print().Msgf("\n") } diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index 6af4268f..a6772794 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration results int errors int + requests int apiKeys []string skipped bool } @@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { @@ -110,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go index 96d21e78..c2328aa9 100644 --- a/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -94,6 +97,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go index cbdcf0f4..6306f121 100644 --- a/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) + s.requests++ resp, err := session.Get(ctx, getUrl, "", map[string]string{ "X-Access-Token": randomApiKey, "User-Agent": "subfinder", }) @@ -106,6 +109,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 198d31e0..2fa3358d 100644 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { + s.requests++ resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) if err != nil && resp == nil { @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go index a2204337..155a06fa 100644 --- a/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -35,6 +35,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index 124686d2..a9040589 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -119,6 +122,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 388cc301..4b015dd6 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -75,6 +75,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -83,6 +84,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -136,6 +138,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers[orgIDHeader] = randomApiKey.orgID } + s.requests++ resp, err := session.HTTPRequest( ctx, http.MethodPost, @@ -220,6 +223,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index 35118ff6..e27a1cbf 100644 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Authorization": "Bearer " + randomApiKey} cookies := "" + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -88,6 +91,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) + s.requests++ resp, err := session.Get(ctx, reqURL, cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -152,6 +156,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go index 489e3173..8b7c1c99 100644 --- a/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) } chaosClient := chaos.New(randomApiKey) + s.requests++ for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { @@ -86,6 +89,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index 2fd6c954..20e6206d 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -103,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index b1dda491..b2ce3feb 100644 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, indexURL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -117,6 +120,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } @@ -128,6 +132,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se return false default: var headers = map[string]string{"Host": "index.commoncrawl.org"} + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index caea79c4..552756c4 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -145,6 +147,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -212,6 +215,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go index 8a89e686..54a158c9 100644 --- a/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -34,6 +35,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -48,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + s.requests++ resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go index 91b3ca5b..ec51b867 100644 --- a/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -32,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { @@ -99,6 +102,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go index 810d8a05..b7464966 100644 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results uint64 + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -70,6 +72,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "application/x-ndjson", } + s.requests++ offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -92,6 +95,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } url := urlTemplate + queryParams.Encode() + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -195,6 +199,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: int(s.results), + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 8062a495..f4c84b6d 100644 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -25,6 +25,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -103,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go index f7faaaac..cb8c34fa 100644 --- a/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se token := randomApiInfo[0] apiKey := randomApiInfo[1] + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go index 800727d7..8f4fd2bd 100644 --- a/pkg/subscraping/sources/domainsproject/domainsproject.go +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) + s.requests++ resp, err := session.HTTPRequest( ctx, "GET", @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index fa0d0bcd..2a9b450a 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors atomic.Int32 results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -65,6 +66,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) + s.requests.Store(0) // Waitgroup for subsources var wg sync.WaitGroup @@ -119,6 +121,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } @@ -139,6 +142,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc // Request requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + s.requests.Add(1) resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { // HTTP 204 is not an error from the Driftnet API diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go index f0f37a4e..ba6e9250 100644 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ b/pkg/subscraping/sources/facebook/ctlogs.go @@ -78,6 +78,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -86,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 if len(s.apiKeys) == 0 { s.skipped = true @@ -108,6 +110,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.Get(ctx, domainsURL, "", nil) if err != nil { s.errors++ @@ -192,6 +195,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go index b885476a..9332e3ec 100644 --- a/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // fofa api doc https://fofa.info/static_pages/api_help qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go index c53206e1..5fc10a01 100644 --- a/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -98,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 69f5e151..63599cd1 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -86,6 +88,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } // Initial request to GitHub search + s.requests++ resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { @@ -167,6 +170,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r default: } + s.requests++ resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -277,6 +281,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index e2a13a11..6325aad3 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -37,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -66,6 +68,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * default: } + s.requests++ resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -91,6 +94,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) + s.requests++ resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { @@ -173,6 +177,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index f85b4fe8..2739a9a9 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + s.requests++ resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -109,5 +112,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go index b3e2e0ec..d27ca4d3 100644 --- a/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -26,6 +26,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -99,6 +102,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index 82ecaeb1..1e26b1f9 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -56,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -85,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -112,6 +115,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -179,6 +183,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index 0d3526ab..91f27d00 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers["api-key"] = randomApiKey } // Request + s.requests++ resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -100,6 +103,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go index aa4881e7..7e9147db 100644 --- a/pkg/subscraping/sources/merklemap/merklemap.go +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -97,6 +99,7 @@ func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[s func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { url := baseURL + "&page=" + strconv.Itoa(page) + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { return nil, err @@ -153,6 +156,7 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index a0c8b86e..8ac65fee 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -40,6 +40,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -47,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,6 +65,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + s.requests++ resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, @@ -120,6 +123,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) + s.requests++ resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, @@ -196,6 +200,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go index a304ebdb..f1f880a6 100644 --- a/pkg/subscraping/sources/onyphe/onyphe.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -36,6 +36,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -73,6 +75,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", url.QueryEscape("category:resolver domain:"+domain), page, pageSize) + s.requests++ resp, err = session.Get(ctx, urlWithQuery, "", headers) if err != nil { @@ -159,6 +162,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go index 698b98ac..3bfd8e62 100644 --- a/pkg/subscraping/sources/profundis/profundis.go +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -52,6 +54,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "text/event-stream", } + s.requests++ resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", headers, bytes.NewReader(requestBody)) @@ -123,5 +126,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go index 85515415..4a0c7e53 100644 --- a/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -71,6 +73,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } apiURL := "https://pugrecon.com/api/v1/domains" + s.requests++ resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -150,5 +153,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go index 2e99fb87..3dd7f8d7 100644 --- a/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/quake/quake.go @@ -36,6 +36,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -69,6 +71,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + s.requests++ resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) @@ -153,5 +156,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go index eb796979..d204a1b3 100644 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -19,6 +19,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -41,6 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -115,5 +118,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go index 61bb593d..67bf651c 100644 --- a/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -102,5 +105,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go index 6463da9a..9e4bae63 100644 --- a/pkg/subscraping/sources/reconeer/reconeer.go +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "X-API-KEY": randomApiKey, } apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) + s.requests++ resp, err := session.Get(ctx, apiURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -106,5 +109,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index 009d5d00..21f8420e 100644 --- a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 pageSize := 1000 go func() { defer func(startTime time.Time) { @@ -56,6 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -82,6 +85,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -150,5 +154,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go index 7855a344..a74ee7d6 100644 --- a/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -15,6 +15,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -22,6 +23,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -91,5 +94,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go index 43d270b0..b2fc70da 100644 --- a/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -24,6 +24,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return default: } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -122,5 +125,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index 0e6846e5..c25700df 100644 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -31,6 +31,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -66,13 +68,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if scrollId == "" { var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) + s.requests++ resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) } if err != nil && ptr.Safe(resp).StatusCode == 403 { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) } @@ -156,5 +161,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index b2195e39..7213602e 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -127,6 +130,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index dde639c4..b96315c3 100644 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba default: } + s.requests++ resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { @@ -110,5 +113,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go index 23806219..f8ebb979 100644 --- a/pkg/subscraping/sources/thc/thc.go +++ b/pkg/subscraping/sources/thc/thc.go @@ -24,6 +24,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,6 +65,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -123,5 +126,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go index 6a6645d3..a788ad53 100644 --- a/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -51,6 +53,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -127,5 +130,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index 831659f6..cde72f99 100644 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run queries the ThreatCrowd API for the given domain and returns found subdomains. @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func(startTime time.Time) { defer func() { @@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Client.Do(req) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -122,5 +125,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 77f5a586..3d25f336 100644 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -93,5 +96,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 446c7c30..812f40e2 100644 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -60,6 +62,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } + s.requests++ resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -125,6 +128,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index ecbe4d94..6f48f058 100644 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -97,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index c4efbdd2..547c3294 100644 --- a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -33,6 +33,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -41,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,6 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -113,5 +116,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go index 92f972c4..b0a13286 100644 --- a/pkg/subscraping/sources/windvane/windvane.go +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -38,6 +38,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -45,6 +46,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -72,6 +74,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var err error requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) + s.requests++ resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", "", headers, bytes.NewReader(requestBody)) @@ -150,5 +153,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 70d491c6..6b10fa34 100644 --- a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -69,6 +71,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se default: } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) + s.requests++ resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { @@ -132,5 +135,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 454ad776..b5fc6c09 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -28,6 +28,7 @@ type BasicAuth struct { // Statistics contains statistics about the scraping process type Statistics struct { TimeTaken time.Duration + Requests int Errors int Results int Skipped bool