diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e74d874b..b22a4444 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -46,6 +46,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" @@ -97,11 +98,12 @@ var AllSources = [...]subscraping.Source{ &whoisxmlapi.Source{}, &zoomeyeapi.Source{}, &facebook.Source{}, - // &threatminer.Source{}, // failing api + // &threatminer.Source{}, // failing api // &reconcloud.Source{}, // failing due to cloudflare bot protection &builtwith.Source{}, &hudsonrock.Source{}, &digitalyama.Source{}, + &submd.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 4febbf9f..dd00fd34 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -57,6 +57,7 @@ var ( "builtwith", "hudsonrock", "digitalyama", + "submd", } expectedDefaultSources = []string{ @@ -94,6 +95,7 @@ var ( // "reconcloud", "builtwith", "digitalyama", + "submd", } expectedDefaultRecursiveSources = []string{ diff --git a/pkg/subscraping/sources/submd/submd.go b/pkg/subscraping/sources/submd/submd.go new file mode 100644 index 00000000..35291938 --- /dev/null +++ b/pkg/subscraping/sources/submd/submd.go @@ -0,0 +1,82 @@ +package submd + +import ( + "bufio" + "context" + "net/http" + "net/url" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + resp, err := s.fetch(ctx, domain, session) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer session.DiscardHTTPResponse(resp) + + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + if line := sc.Text(); line != "" { + for _, sub := range session.Extractor.Extract(line) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub} + s.results++ + } + } + } + if err := sc.Err(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + return results +} + +// fetch issues the API call, upgrades to Bearer auth when a key is available. +func (s *Source) fetch(ctx context.Context, domain string, session *subscraping.Session) (*http.Response, error) { + endpoint := "https://api.sub.md/v1/search?apex=" + url.QueryEscape(domain) + + if len(s.apiKeys) > 0 { + return session.Get(ctx, endpoint, "", map[string]string{ + "Authorization": "Bearer " + subscraping.PickRandom(s.apiKeys, s.Name()), + }) + } + return session.SimpleGet(ctx, endpoint) +} + +func (s *Source) Name() string { return "submd" } +func (s *Source) IsDefault() bool { return true } +func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) NeedsKey() bool { return false } +func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + } +}