admin/projectdiscovery-subfinder
1
0
Fork 0
mirror of https://github.com/projectdiscovery/subfinder synced 2026-06-21 14:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove zoomeye (#943)

Browse Source
This commit is contained in:
Dogan Can Bakir
2023-08-24 15:38:07 +03:00
committed by GitHub
parent 2acb1b3406
commit 1d37afff6f
6 changed files with 6 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-test.yml
-1
View File
@@ -53,7 +53,6 @@ jobs:
THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}}
VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}}
WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}}
ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}}
ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}}
uses: nick-invision/retry@v2
with:
README.md
+5 -5
View File
@@ -66,7 +66,7 @@ SOURCE:
-s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.
-recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)
-all use all sources for enumeration (slow)
-es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeye)
-es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeyeapi)
FILTER:
-m, -match string[] subdomain or list of subdomain to match (file or comma separated)
@@ -121,14 +121,14 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
`subfinder` can be used right after the installation, however the following services require configuring API keys to work:
[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com)
[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com)
You can also use the `subfinder -ls` command to display all the available sources.
These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys
can be specified for each of these services from which one of them will be used for enumeration.
Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix`, `360quake` and `ZoomEye`, need to be separated with a colon (`:`).
Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `360quake`, need to be separated with a colon (`:`).
An example provider config file:
@@ -147,8 +147,8 @@ shodan:
github:
- ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X
- ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4
zoomeye:
- ZOOMEYE_USERNAME:ZOOMEYE_PASSWORD
zoomeyeapi:
- 4f73021d-ff95-4f53-937f-83d6db719eec
quake:
- 0cb9030c-0a40-48a3-b8c4-fca28e466ba3
facebook:
v2/pkg/passive/sources.go
-2
View File
@@ -45,7 +45,6 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
)
@@ -86,7 +85,6 @@ var AllSources = [...]subscraping.Source{
&virustotal.Source{},
&waybackarchive.Source{},
&whoisxmlapi.Source{},
&zoomeye.Source{},
&zoomeyeapi.Source{},
&facebook.Source{},
// &threatminer.Source{}, // failing api
v2/pkg/passive/sources_test.go
-1
View File
@@ -45,7 +45,6 @@ var (
"virustotal",
"waybackarchive",
"whoisxmlapi",
"zoomeye",
"zoomeyeapi",
"hunter",
"leakix",
v2/pkg/runner/options.go
+1 -1
View File
@@ -89,7 +89,7 @@ func ParseOptions() *Options {
flagSet.StringSliceVarP(&options.Sources, "sources", "s", nil, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions),
flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)"),
flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"),
flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeyeapi)", goflags.NormalizedStringSliceOptions),
)
flagSet.CreateGroup("filter", "Filter",
v2/pkg/subscraping/sources/zoomeye/zoomeye.go
-179
View File
@@ -1,179 +0,0 @@
// Package zoomeye logic
package zoomeye
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// zoomAuth holds the ZoomEye credentials
type zoomAuth struct {
User string `json:"username"`
Pass string `json:"password"`
}
type loginResp struct {
JWT string `json:"access_token"`
}
// search results
type zoomeyeResults struct {
Matches []struct {
Site string `json:"site"`
Domains []string `json:"domains"`
} `json:"matches"`
}
// Source is the passive scraping agent
type Source struct {
apiKeys []apiKey
timeTaken time.Duration
errors int
results int
skipped bool
}
type apiKey struct {
username string
password string
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey.username == "" || randomApiKey.password == "" {
s.skipped = true
return
}
jwt, err := doLogin(ctx, session, randomApiKey)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
// check if jwt is null
if jwt == "" {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye"),
}
s.errors++
return
}
headers := map[string]string{
"Authorization": fmt.Sprintf("JWT %s", jwt),
"Accept": "application/json",
"Content-Type": "application/json",
}
for currentPage := 0; currentPage <= 100; currentPage++ {
api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage)
resp, err := session.Get(ctx, api, "", headers)
isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
if err != nil {
if !isForbidden && currentPage == 0 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
}
return
}
var res zoomeyeResults
err = json.NewDecoder(resp.Body).Decode(&res)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
resp.Body.Close()
return
}
resp.Body.Close()
for _, r := range res.Matches {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site}
s.results++
for _, domain := range r.Domains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}
s.results++
}
}
}
}()
return results
}
// doLogin performs authentication on the ZoomEye API
func doLogin(ctx context.Context, session *subscraping.Session, randomApiKey apiKey) (string, error) {
creds := &zoomAuth{
User: randomApiKey.username,
Pass: randomApiKey.password,
}
body, err := json.Marshal(&creds)
if err != nil {
return "", err
}
resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body))
if err != nil {
session.DiscardHTTPResponse(resp)
return "", err
}
defer resp.Body.Close()
var login loginResp
err = json.NewDecoder(resp.Body).Decode(&login)
if err != nil {
return "", err
}
return login.JWT, nil
}
// Name returns the name of the source
func (s *Source) Name() string {
return "zoomeye"
}
func (s *Source) IsDefault() bool {
return false
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) NeedsKey() bool {
return true
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
return apiKey{k, v}
})
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}