feat: add shodanct passive source (#1796)

* feat: add shodanct passive source

* fix lint

---------

Co-authored-by: Mzack9999 <mzack9999@protonmail.com>

pkg/passive/sources.go

@@ -48,6 +48,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodanct"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc"
@@ -104,6 +105,7 @@ var AllSources = [...]subscraping.Source{
 	&rsecloud.Source{},
 	&securitytrails.Source{},
 	&shodan.Source{},
+	&shodanct.Source{},
 	&sitedossier.Source{},
 	&thc.Source{},
 	&threatbook.Source{},

pkg/passive/sources_test.go

@@ -45,6 +45,7 @@ var (
 		"securitytrails",
 		"profundis",
 		"shodan",
+		"shodanct",
 		"sitedossier",
 		"threatbook",
 		"threatcrowd",
@@ -95,6 +96,7 @@ var (
 		"securitytrails",
 		"profundis",
 		"shodan",
+		"shodanct",
 		"windvane",
 		"virustotal",
 		"whoisxmlapi",
@@ -119,6 +121,7 @@ var (
 		"driftnet",
 		"hackertarget",
 		"securitytrails",
+		"shodanct",
 		"virustotal",
 		"leakix",
 		"merklemap",

pkg/runner/util.go

@@ -6,12 +6,11 @@ import (
 )
 
 func loadFromFile(file string) ([]string, error) {
-	chanItems, err := fileutil.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
 	var items []string
-	for item := range chanItems {
+	for item, err := range fileutil.Lines(file) {
+		if err != nil {
+			return nil, err
+		}
 		item = preprocessDomain(item)
 		if item == "" {
 			continue

pkg/subscraping/sources/shodanct/shodanct.go

@@ -0,0 +1,112 @@
+// Package shodanct logic
+package shodanct
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+// Source is the passive scraping agent
+type Source struct {
+	timeTaken time.Duration
+	errors    int
+	results   int
+	requests  int
+}
+
+// Run function returns all subdomains found with the service
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+	s.errors = 0
+	s.results = 0
+	s.requests = 0
+
+	go func() {
+		defer func(startTime time.Time) {
+			s.timeTaken = time.Since(startTime)
+			close(results)
+		}(time.Now())
+
+		searchURL := fmt.Sprintf("https://ctl.shodan.io/api/v1/domain/%s/hostnames", domain)
+		s.requests++
+		resp, err := session.SimpleGet(ctx, searchURL)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			results <- subscraping.Result{
+				Source: s.Name(), Type: subscraping.Error,
+				Error: fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, searchURL),
+			}
+			s.errors++
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+
+		defer session.DiscardHTTPResponse(resp)
+
+		var hostnames []string
+		if err := jsoniter.NewDecoder(resp.Body).Decode(&hostnames); err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			return
+		}
+
+		for _, hostname := range hostnames {
+			for _, subdomain := range session.Extractor.Extract(hostname) {
+				select {
+				case <-ctx.Done():
+					return
+				case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
+					s.results++
+				}
+			}
+		}
+	}()
+
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "shodanct"
+}
+
+func (s *Source) IsDefault() bool {
+	return true
+}
+
+func (s *Source) HasRecursiveSupport() bool {
+	return true
+}
+
+func (s *Source) KeyRequirement() subscraping.KeyRequirement {
+	return subscraping.NoKey
+}
+
+func (s *Source) NeedsKey() bool {
+	return s.KeyRequirement() == subscraping.RequiredKey
+}
+
+func (s *Source) AddApiKeys(_ []string) {
+	// no key needed
+}
+
+func (s *Source) Statistics() subscraping.Statistics {
+	return subscraping.Statistics{
+		Errors:    s.errors,
+		Results:   s.results,
+		Requests:  s.requests,
+		TimeTaken: s.timeTaken,
+	}
+}