diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index 45fd53f1..27de0210 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -48,6 +48,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodanct" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" @@ -104,6 +105,7 @@ var AllSources = [...]subscraping.Source{ &rsecloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, + &shodanct.Source{}, &sitedossier.Source{}, &thc.Source{}, &threatbook.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index bd44d408..8deeda39 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -45,6 +45,7 @@ var ( "securitytrails", "profundis", "shodan", + "shodanct", "sitedossier", "threatbook", "threatcrowd", @@ -95,6 +96,7 @@ var ( "securitytrails", "profundis", "shodan", + "shodanct", "windvane", "virustotal", "whoisxmlapi", @@ -119,6 +121,7 @@ var ( "driftnet", "hackertarget", "securitytrails", + "shodanct", "virustotal", "leakix", "merklemap", diff --git a/pkg/runner/util.go b/pkg/runner/util.go index f0458c73..ba062fcb 100644 --- a/pkg/runner/util.go +++ b/pkg/runner/util.go @@ -6,12 +6,11 @@ import ( ) func loadFromFile(file string) ([]string, error) { - chanItems, err := fileutil.ReadFile(file) - if err != nil { - return nil, err - } var items []string - for item := range chanItems { + for item, err := range fileutil.Lines(file) { + if err != nil { + return nil, err + } item = preprocessDomain(item) if item == "" { continue diff --git a/pkg/subscraping/sources/shodanct/shodanct.go b/pkg/subscraping/sources/shodanct/shodanct.go new file mode 100644 index 00000000..49de1258 --- /dev/null +++ b/pkg/subscraping/sources/shodanct/shodanct.go @@ -0,0 +1,112 @@ +// Package shodanct logic +package shodanct + +import ( + "context" + "fmt" + "net/http" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int + requests int +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + searchURL := fmt.Sprintf("https://ctl.shodan.io/api/v1/domain/%s/hostnames", domain) + s.requests++ + resp, err := session.SimpleGet(ctx, searchURL) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + if resp.StatusCode != http.StatusOK { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, + Error: fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, searchURL), + } + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer session.DiscardHTTPResponse(resp) + + var hostnames []string + if err := jsoniter.NewDecoder(resp.Body).Decode(&hostnames); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, hostname := range hostnames { + for _, subdomain := range session.Extractor.Extract(hostname) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "shodanct" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + } +}