mirror of
https://github.com/projectdiscovery/subfinder
synced 2026-06-21 14:05:24 +00:00
Dogan Can Bakir
2a8ce2f3e6
* feat: add shodanct passive source * fix lint --------- Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
205 lines
7.6 KiB
Go
205 lines
7.6 KiB
Go
package passive
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
|
|
"golang.org/x/exp/maps"
|
|
|
|
"github.com/projectdiscovery/gologger"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodanct"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane"
|
|
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
|
|
mapsutil "github.com/projectdiscovery/utils/maps"
|
|
)
|
|
|
|
var AllSources = [...]subscraping.Source{
|
|
&alienvault.Source{},
|
|
&anubis.Source{},
|
|
&bevigil.Source{},
|
|
&bufferover.Source{},
|
|
&builtwith.Source{},
|
|
&c99.Source{},
|
|
&censys.Source{},
|
|
&certspotter.Source{},
|
|
&chaos.Source{},
|
|
&chinaz.Source{},
|
|
&commoncrawl.Source{},
|
|
&crtsh.Source{},
|
|
&digitalyama.Source{},
|
|
&digitorus.Source{},
|
|
&dnsdb.Source{},
|
|
&dnsdumpster.Source{},
|
|
&dnsrepo.Source{},
|
|
&domainsproject.Source{},
|
|
&driftnet.Source{},
|
|
&fofa.Source{},
|
|
&fullhunt.Source{},
|
|
&github.Source{},
|
|
&hackertarget.Source{},
|
|
&hudsonrock.Source{},
|
|
&intelx.Source{},
|
|
&leakix.Source{},
|
|
&merklemap.Source{},
|
|
&netlas.Source{},
|
|
&onyphe.Source{},
|
|
&profundis.Source{},
|
|
&pugrecon.Source{},
|
|
&quake.Source{},
|
|
&rapiddns.Source{},
|
|
// &reconcloud.Source{}, // failing due to cloudflare bot protection
|
|
&reconeer.Source{},
|
|
&redhuntlabs.Source{},
|
|
// &riddler.Source{}, // failing due to cloudfront protection
|
|
&robtex.Source{},
|
|
&rsecloud.Source{},
|
|
&securitytrails.Source{},
|
|
&shodan.Source{},
|
|
&shodanct.Source{},
|
|
&sitedossier.Source{},
|
|
&thc.Source{},
|
|
&threatbook.Source{},
|
|
&threatcrowd.Source{},
|
|
// &threatminer.Source{}, // failing api
|
|
&urlscan.Source{},
|
|
&virustotal.Source{},
|
|
&waybackarchive.Source{},
|
|
&whoisxmlapi.Source{},
|
|
&windvane.Source{},
|
|
&zoomeyeapi.Source{},
|
|
&submd.Source{},
|
|
}
|
|
|
|
var sourceWarnings = mapsutil.NewSyncLockMap[string, string](
|
|
mapsutil.WithMap(mapsutil.Map[string, string]{}))
|
|
|
|
var NameSourceMap = make(map[string]subscraping.Source, len(AllSources))
|
|
|
|
func init() {
|
|
for _, currentSource := range AllSources {
|
|
NameSourceMap[strings.ToLower(currentSource.Name())] = currentSource
|
|
}
|
|
}
|
|
|
|
// Agent is a struct for running passive subdomain enumeration
|
|
// against a given host. It wraps subscraping package and provides
|
|
// a layer to build upon.
|
|
type Agent struct {
|
|
sources []subscraping.Source
|
|
}
|
|
|
|
// New creates a new agent for passive subdomain discovery
|
|
func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSupportingRecurse bool) *Agent {
|
|
sources := make(map[string]subscraping.Source, len(AllSources))
|
|
|
|
if useAllSources {
|
|
maps.Copy(sources, NameSourceMap)
|
|
} else {
|
|
if len(sourceNames) > 0 {
|
|
for _, source := range sourceNames {
|
|
if NameSourceMap[source] == nil {
|
|
gologger.Warning().Msgf("There is no source with the name: %s", source)
|
|
} else {
|
|
sources[source] = NameSourceMap[source]
|
|
}
|
|
}
|
|
} else {
|
|
for _, currentSource := range AllSources {
|
|
if currentSource.IsDefault() {
|
|
sources[currentSource.Name()] = currentSource
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(excludedSourceNames) > 0 {
|
|
for _, sourceName := range excludedSourceNames {
|
|
delete(sources, sourceName)
|
|
}
|
|
}
|
|
|
|
if useSourcesSupportingRecurse {
|
|
for sourceName, source := range sources {
|
|
if !source.HasRecursiveSupport() {
|
|
delete(sources, sourceName)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(sources) == 0 {
|
|
gologger.Fatal().Msg("No sources selected for this search")
|
|
}
|
|
|
|
gologger.Debug().Msgf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", "))
|
|
|
|
for _, currentSource := range sources {
|
|
if warning, ok := sourceWarnings.Get(strings.ToLower(currentSource.Name())); ok {
|
|
gologger.Warning().Msg(warning)
|
|
}
|
|
}
|
|
|
|
for _, source := range sources {
|
|
keyReq := source.KeyRequirement()
|
|
if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey {
|
|
if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" {
|
|
source.AddApiKeys([]string{apiKey})
|
|
}
|
|
}
|
|
}
|
|
|
|
// Create the agent, insert the sources and remove the excluded sources
|
|
agent := &Agent{sources: maps.Values(sources)}
|
|
|
|
return agent
|
|
}
|