admin/projectdiscovery-subfinder
1
0
Fork 0
mirror of https://github.com/projectdiscovery/subfinder synced 2026-06-21 14:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1549 from projectdiscovery/dev

Browse Source 
v2.7.0
This commit is contained in:
Dogan Can Bakir
2025-02-27 10:12:57 +03:00
committed by GitHub
parent eaefdec9d7 aa7fd52c9b
commit 0d34c55002
15 changed files with 312 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-test.yml
-1
View File
@@ -51,7 +51,6 @@ jobs:
HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}}
INTELX_API_KEY: ${{secrets.INTELX_API_KEY}}
LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}}
PASSIVETOTAL_API_KEY: ${{secrets.PASSIVETOTAL_API_KEY}}
QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}}
ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}}
SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}}
v2/go.mod
+10 -10
View File
@@ -8,12 +8,12 @@ require (
github.com/json-iterator/go v1.1.12
github.com/lib/pq v1.10.9
github.com/projectdiscovery/chaos-client v0.5.2
github.com/projectdiscovery/dnsx v1.2.1
github.com/projectdiscovery/dnsx v1.2.2
github.com/projectdiscovery/fdmax v0.0.4
github.com/projectdiscovery/gologger v1.1.40
github.com/projectdiscovery/ratelimit v0.0.68
github.com/projectdiscovery/retryablehttp-go v1.0.95
github.com/projectdiscovery/utils v0.4.6
github.com/projectdiscovery/gologger v1.1.44
github.com/projectdiscovery/ratelimit v0.0.70
github.com/projectdiscovery/retryablehttp-go v1.0.99
github.com/projectdiscovery/utils v0.4.11
github.com/rs/xid v1.5.0
github.com/stretchr/testify v1.9.0
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@@ -71,10 +71,10 @@ require (
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/cdncheck v1.1.0 // indirect
github.com/projectdiscovery/fastdialer v0.2.14 // indirect
github.com/projectdiscovery/hmap v0.0.74 // indirect
github.com/projectdiscovery/fastdialer v0.3.0 // indirect
github.com/projectdiscovery/hmap v0.0.80 // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
github.com/projectdiscovery/networkpolicy v0.0.9 // indirect
github.com/projectdiscovery/networkpolicy v0.1.1 // indirect
github.com/refraction-networking/utls v1.6.7 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
@@ -124,8 +124,8 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/goflags v0.1.64
github.com/projectdiscovery/retryabledns v1.0.93 // indirect
github.com/projectdiscovery/goflags v0.1.72
github.com/projectdiscovery/retryabledns v1.0.94 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sys v0.28.0 // indirect
)
v2/go.sum
+22 -21
View File
@@ -194,38 +194,39 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU=
github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU=
github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg=
github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0=
github.com/projectdiscovery/fastdialer v0.2.14 h1:/cndy+5celjoYzbk4LksHYOCTpFGIJY8RF/EK31Opjs=
github.com/projectdiscovery/fastdialer v0.2.14/go.mod h1:z5yKQ/YWaVrBMfdL6f5J7VytUx9wxc5vs/Lf51QelCw=
github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg=
github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I=
github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk=
github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ=
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
github.com/projectdiscovery/gologger v1.1.40 h1:FSIhKnYKzuIEIz3RTg6JX9JtDKgkEzIEf2v5RYckoQ4=
github.com/projectdiscovery/gologger v1.1.40/go.mod h1:8AUxYXmClqOWJgZ5wknNn5rRK3UlrXQ/r9JjX+gp5Gg=
github.com/projectdiscovery/hmap v0.0.74 h1:j0TpS9fJxisfdGcIRjaZ3qgyjP3pBkRJf12ZWv64rOI=
github.com/projectdiscovery/hmap v0.0.74/go.mod h1:qEPAdq/gWQU/IEI+QMzSyL+HYdqayR64V9vGTI/W38c=
github.com/projectdiscovery/goflags v0.1.72 h1:tSR+BnfDLbfTGYYVg4k1oQcFOoYXPY1pllV0MHtx3ek=
github.com/projectdiscovery/goflags v0.1.72/go.mod h1:C2cZ+PJRx7bbArEp/qFUixjsYFDd3etFNNHMUdJqfr8=
github.com/projectdiscovery/gologger v1.1.44 h1:tprWkKzKt37pz4HG2tvhzrOCQNIn8A3CEki6BRzXE5o=
github.com/projectdiscovery/gologger v1.1.44/go.mod h1:ZQS0eJq7BwKM0xxFqwZFUkAH1bkIqe90EOFBP4LENH4=
github.com/projectdiscovery/hmap v0.0.80 h1:2PSo3qQNKanK6i6DF4NzsVEJANe6tMIBmBtxvF4AKK8=
github.com/projectdiscovery/hmap v0.0.80/go.mod h1:YmZ9qwtl7MDJYrpxJ+MEw4N4V58w18WGPsQHgpdIV0s=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4=
github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw=
github.com/projectdiscovery/retryabledns v1.0.93 h1:iKcEEEH77WwUf5EGimhHxCDdqBF2kOl7WhQi3VQXB8Q=
github.com/projectdiscovery/retryabledns v1.0.93/go.mod h1:f5HmPdVr3CUm4tHHiB0UyiZVQTYYAKTqfoj8M2gCvqo=
github.com/projectdiscovery/retryablehttp-go v1.0.95 h1:5CHhWLMovX1MD9W3HzlsMBY3xA+dyeqta2gSWo3j92E=
github.com/projectdiscovery/retryablehttp-go v1.0.95/go.mod h1:/7CHaD7vqnqBD++AI0JsJdcYyq1Wbf4vMhddjy7sZjI=
github.com/projectdiscovery/utils v0.4.6 h1:lwbS5d/f70wyDwuwF6lAVkn390hEI/0LOtqyqJEI+qE=
github.com/projectdiscovery/utils v0.4.6/go.mod h1:eevtW7+x7ydrBdmOenmHdqqJKRv3VqY2QUR7vs4qRfU=
github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU=
github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU=
github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0vd8ER+eCHQjc=
github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM=
github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0=
github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak=
github.com/projectdiscovery/retryablehttp-go v1.0.99 h1:S+lQqo1ZnO5aoWsBV8HapGslJSaYVUII954SnH1RSjw=
github.com/projectdiscovery/retryablehttp-go v1.0.99/go.mod h1:8Mv9L9vjmam16garE6/dqLFkT0ZcfLNSo9O1zFBiPlE=
github.com/projectdiscovery/utils v0.4.11 h1:MWqCFxYINQPa4KWMRNah7W0N1COGRhqOpGVhiR/VaO0=
github.com/projectdiscovery/utils v0.4.11/go.mod h1:47tvqErksJELcxDBH8An2i9qvUe5E1qR7B72xxqiyqU=
github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
v2/pkg/passive/sources.go
+5 -5
View File
@@ -34,7 +34,6 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs"
@@ -43,10 +42,12 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama"
mapsutil "github.com/projectdiscovery/utils/maps"
)
@@ -75,7 +76,6 @@ var AllSources = [...]subscraping.Source{
&intelx.Source{},
&netlas.Source{},
&leakix.Source{},
&passivetotal.Source{},
&quake.Source{},
&rapiddns.Source{},
&redhuntlabs.Source{},
@@ -85,6 +85,7 @@ var AllSources = [...]subscraping.Source{
&shodan.Source{},
&sitedossier.Source{},
&threatbook.Source{},
&threatcrowd.Source{},
&virustotal.Source{},
&waybackarchive.Source{},
&whoisxmlapi.Source{},
@@ -94,12 +95,11 @@ var AllSources = [...]subscraping.Source{
// &reconcloud.Source{}, // failing due to cloudflare bot protection
&builtwith.Source{},
&hudsonrock.Source{},
&digitalyama.Source{},
}
var sourceWarnings = mapsutil.NewSyncLockMap[string, string](
mapsutil.WithMap(mapsutil.Map[string, string]{
"passivetotal": "New API credentials for PassiveTotal can't be generated, but existing user account credentials are still functional. Please ensure your integrations are using valid credentials.",
}))
mapsutil.WithMap(mapsutil.Map[string, string]{}))
var NameSourceMap = make(map[string]subscraping.Source, len(AllSources))
v2/pkg/passive/sources_test.go
+3 -3
View File
@@ -33,7 +33,6 @@ var (
"hackertarget",
"intelx",
"netlas",
"passivetotal",
"quake",
"rapiddns",
"redhuntlabs",
@@ -43,6 +42,7 @@ var (
"shodan",
"sitedossier",
"threatbook",
"threatcrowd",
"virustotal",
"waybackarchive",
"whoisxmlapi",
@@ -54,6 +54,7 @@ var (
// "reconcloud",
"builtwith",
"hudsonrock",
"digitalyama",
}
expectedDefaultSources = []string{
@@ -74,7 +75,6 @@ var (
"fullhunt",
"hackertarget",
"intelx",
"passivetotal",
"quake",
"redhuntlabs",
"robtex",
@@ -89,6 +89,7 @@ var (
// "threatminer",
// "reconcloud",
"builtwith",
"digitalyama",
}
expectedDefaultRecursiveSources = []string{
@@ -100,7 +101,6 @@ var (
"dnsdb",
"digitorus",
"hackertarget",
"passivetotal",
"securitytrails",
"virustotal",
"leakix",
v2/pkg/runner/banners.go
+1 -1
View File
@@ -17,7 +17,7 @@ const banner = `
const ToolName = `subfinder`
// Version is the current version of subfinder
const version = `v2.6.8`
const version = `v2.7.0`
// showBanner is used to show the banner to the user
func showBanner() {
v2/pkg/runner/enumerate.go
+1
View File
@@ -19,6 +19,7 @@ import (
const maxNumCount = 2
var replacer = strings.NewReplacer(
"/", "",
"•.", "",
"•", "",
"*.", "",
v2/pkg/runner/runner.go
+1
View File
@@ -119,6 +119,7 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.
ip, _ := regexp.Compile(`^([0-9\.]+$)`)
for scanner.Scan() {
domain := preprocessDomain(scanner.Text())
domain = replacer.Replace(domain)
if domain == "" || (r.options.ExcludeIps && ip.MatchString(domain)) {
continue
v2/pkg/subscraping/sources/anubis/anubis.go
+6
View File
@@ -4,6 +4,7 @@ package anubis
import (
"context"
"fmt"
"net/http"
"time"
jsoniter "github.com/json-iterator/go"
@@ -38,6 +39,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return
}
var subdomains []string
err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
if err != nil {
v2/pkg/subscraping/sources/digitalyama/digitalyama.go
+128
View File
@@ -0,0 +1,128 @@
package digitalyama
import (
"context"
"fmt"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
skipped bool
}
type digitalYamaResponse struct {
Query string `json:"query"`
Count int `json:"count"`
Subdomains []string `json:"subdomains"`
UsageSummary struct {
QueryCost float64 `json:"query_cost"`
CreditsRemaining float64 `json:"credits_remaining"`
} `json:"usage_summary"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain)
resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
var errResponse struct {
Detail []struct {
Loc []string `json:"loc"`
Msg string `json:"msg"`
Type string `json:"type"`
} `json:"detail"`
}
err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)}
s.errors++
return
}
if len(errResponse.Detail) > 0 {
errMsg := errResponse.Detail[0].Msg
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)}
} else {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)}
}
s.errors++
return
}
var response digitalYamaResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
for _, subdomain := range response.Subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "digitalyama"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) NeedsKey() bool {
return true
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go
+12 -2
View File
@@ -21,7 +21,7 @@ type Source struct {
}
type DnsRepoResponse []struct {
Domain string
Domain string `json:"domain"`
}
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
@@ -40,7 +40,17 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
s.skipped = true
return
}
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain))
randomApiInfo := strings.Split(randomApiKey, ":")
if len(randomApiInfo) != 2 {
s.skipped = true
return
}
token := randomApiInfo[0]
apiKey := randomApiInfo[1]
resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
v2/pkg/subscraping/sources/netlas/netlas.go
+5 -5
View File
@@ -98,10 +98,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
// Make a single POST request to get all domains via download method
apiUrl := "https://app.netlas.io/api/domains/download/"
query := fmt.Sprintf("domain:(domain:*.%s AND NOT domain:%s)", domain, domain)
query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain)
requestBody := map[string]interface{}{
"q": query,
"fields": []string{"*"},
"q": query,
"fields": []string{"*"},
"source_type": "include",
"size": domainsCount.Count,
}
@@ -116,8 +116,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name())
resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{
"accept": "application/json",
"X-API-Key": randomApiKey,
"accept": "application/json",
"X-API-Key": randomApiKey,
"Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
v2/pkg/subscraping/sources/passivetotal/passivetotal.go
-126
View File
@@ -1,126 +0,0 @@
// Package passivetotal logic
package passivetotal
import (
"bytes"
"context"
"regexp"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
var passiveTotalFilterRegex = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`)
type response struct {
Subdomains []string `json:"subdomains"`
}
// Source is the passive scraping agent
type Source struct {
apiKeys []apiKey
timeTaken time.Duration
errors int
results int
skipped bool
}
type apiKey struct {
username string
password string
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey.username == "" || randomApiKey.password == "" {
s.skipped = true
return
}
// Create JSON Get body
var request = []byte(`{"query":"` + domain + `"}`)
resp, err := session.HTTPRequest(
ctx,
"GET",
"https://api.passivetotal.org/v2/enrichment/subdomains",
"",
map[string]string{"Content-Type": "application/json"},
bytes.NewBuffer(request),
subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password},
)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var data response
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
resp.Body.Close()
return
}
resp.Body.Close()
for _, subdomain := range data.Subdomains {
// skip entries like xxx.xxx.xxx.xxx\032domain.tld
if passiveTotalFilterRegex.MatchString(subdomain) {
continue
}
finalSubdomain := subdomain + "." + domain
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain}
s.results++
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "passivetotal"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) NeedsKey() bool {
return true
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
return apiKey{k, v}
})
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go
+117
View File
@@ -0,0 +1,117 @@
package threatcrowd
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// threatCrowdResponse represents the JSON response from the ThreatCrowd API.
type threatCrowdResponse struct {
ResponseCode string `json:"response_code"`
Subdomains []string `json:"subdomains"`
Undercount string `json:"undercount"`
}
// Source implements the subscraping.Source interface for ThreatCrowd.
type Source struct {
timeTaken time.Duration
errors int
results int
}
// Run queries the ThreatCrowd API for the given domain and returns found subdomains.
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
go func(startTime time.Time) {
defer func() {
s.timeTaken = time.Since(startTime)
close(results)
}()
url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
resp, err := session.Client.Do(req)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)}
s.errors++
return
}
body, err := io.ReadAll(resp.Body)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
var tcResponse threatCrowdResponse
if err := json.Unmarshal(body, &tcResponse); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
for _, subdomain := range tcResponse.Subdomains {
if subdomain != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
}
}(time.Now())
return results
}
// Name returns the name of the source.
func (s *Source) Name() string {
return "threatcrowd"
}
// IsDefault indicates whether this source is enabled by default.
func (s *Source) IsDefault() bool {
return false
}
// HasRecursiveSupport indicates if the source supports recursive searches.
func (s *Source) HasRecursiveSupport() bool {
return false
}
// NeedsKey indicates if the source requires an API key.
func (s *Source) NeedsKey() bool {
return false
}
// AddApiKeys is a no-op since ThreatCrowd does not require an API key.
func (s *Source) AddApiKeys(_ []string) {}
// Statistics returns usage statistics.
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
}
}
v2/pkg/subscraping/sources/virustotal/virustotal.go
+1 -1
View File
@@ -51,7 +51,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
var cursor string = ""
for {
var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain)
var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain)
if cursor != "" {
url = fmt.Sprintf("%s&cursor=%s", url, cursor)
}