diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index 54f8534c..2f1cdb11 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -55,6 +55,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ session.DiscardHTTPResponse(resp) return } diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index 7e2045aa..5e545b40 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -50,32 +50,30 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } body, err := io.ReadAll(resp.Body) - session.DiscardHTTPResponse(resp) - - SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") - - if SubdomainList.ToBool() { - _data := []byte(SubdomainList.ToString()) - for i := 0; i < SubdomainList.Size(); i++ { - select { - case <-ctx.Done(): - return - default: - } - subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() - select { - case <-ctx.Done(): - return - case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: - s.results++ - } - } - } else { + if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } + + SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + if SubdomainList.ValueType() != jsoniter.ArrayValue { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("chinaz: unexpected response shape, missing Result.ContributingSubdomainList")} + s.errors++ + return + } + + _data := []byte(SubdomainList.ToString()) + for i := 0; i < SubdomainList.Size(); i++ { + subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } }() return results diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index 2c00e018..579f37d5 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -39,8 +39,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) } - htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) - s.requests++ resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index 560f4fe6..69331359 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "fmt" - "io" "time" jsoniter "github.com/json-iterator/go" @@ -131,14 +130,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - - _, err = io.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } session.DiscardHTTPResponse(resp) status = response.Status diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index f5f3d365..6e3cbcf9 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -60,6 +60,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ session.DiscardHTTPResponse(resp) return }