Merge pull request #1777 from x-stp/fix/mortal-combat-round-one

fix(sources): hackertarget, intelx, shodan, c99, chinaz control-flow bugs

pkg/subscraping/sources/c99/c99.go

@@ -55,6 +55,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		s.requests++
 		resp, err := session.SimpleGet(ctx, searchURL)
 		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
 			session.DiscardHTTPResponse(resp)
 			return
 		}

pkg/subscraping/sources/chinaz/chinaz.go

@@ -50,32 +50,30 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		}
 
 		body, err := io.ReadAll(resp.Body)
-
 		session.DiscardHTTPResponse(resp)
-
-		SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
-
-		if SubdomainList.ToBool() {
-			_data := []byte(SubdomainList.ToString())
-			for i := 0; i < SubdomainList.Size(); i++ {
-				select {
-				case <-ctx.Done():
-					return
-				default:
-				}
-				subdomain := jsoniter.Get(_data, i, "DataUrl").ToString()
-				select {
-				case <-ctx.Done():
-					return
-				case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
-					s.results++
-				}
-			}
-		} else {
+		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 			s.errors++
 			return
 		}
+
+		SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList")
+		if SubdomainList.ValueType() != jsoniter.ArrayValue {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("chinaz: unexpected response shape, missing Result.ContributingSubdomainList")}
+			s.errors++
+			return
+		}
+
+		_data := []byte(SubdomainList.ToString())
+		for i := 0; i < SubdomainList.Size(); i++ {
+			subdomain := jsoniter.Get(_data, i, "DataUrl").ToString()
+			select {
+			case <-ctx.Done():
+				return
+			case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
+				s.results++
+			}
+		}
 	}()
 
 	return results

pkg/subscraping/sources/hackertarget/hackertarget.go

@@ -39,8 +39,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey)
 		}
 
-		htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey)
-
 		s.requests++
 		resp, err := session.SimpleGet(ctx, htSearchUrl)
 		if err != nil {

pkg/subscraping/sources/intelx/intelx.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -131,14 +130,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 				session.DiscardHTTPResponse(resp)
 				return
 			}
-
-			_, err = io.ReadAll(resp.Body)
-			if err != nil {
-				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-				s.errors++
-				session.DiscardHTTPResponse(resp)
-				return
-			}
 			session.DiscardHTTPResponse(resp)
 
 			status = response.Status

pkg/subscraping/sources/shodan/shodan.go

@@ -60,6 +60,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			s.requests++
 			resp, err := session.SimpleGet(ctx, searchURL)
 			if err != nil {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				s.errors++
 				session.DiscardHTTPResponse(resp)
 				return
 			}