diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index b70b6d14..64f4dd8e 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -48,6 +48,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" @@ -108,6 +109,7 @@ var AllSources = [...]subscraping.Source{ &builtwith.Source{}, &hudsonrock.Source{}, &digitalyama.Source{}, + &thc.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 9c2676b0..a1f99be8 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -60,6 +60,7 @@ var ( "builtwith", "hudsonrock", "digitalyama", + "thc", } expectedDefaultSources = []string{ @@ -100,6 +101,7 @@ var ( // "reconcloud", "builtwith", "digitalyama", + "thc", } expectedDefaultRecursiveSources = []string{ diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go new file mode 100644 index 00000000..23806219 --- /dev/null +++ b/pkg/subscraping/sources/thc/thc.go @@ -0,0 +1,127 @@ +// Package thc logic +package thc + +import ( + "bytes" + "context" + "encoding/json" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Domains []struct { + Domain string `json:"domain"` + } `json:"domains"` + NextPageState string `json:"next_page_state"` +} + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int + skipped bool +} + +type requestBody struct { + Domain string `json:"domain"` + PageState string `json:"page_state"` + Limit int `json:"limit"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + var pageState string + headers := map[string]string{"Content-Type": "application/json"} + apiURL := "https://ip.thc.org/api/v1/lookup/subdomains" + + for { + reqBody := requestBody{ + Domain: domain, + PageState: pageState, + Limit: 1000, + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var thcResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, domainRecord := range thcResponse.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain} + s.results++ + } + + pageState = thcResponse.NextPageState + + if pageState == "" { + break + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "thc" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // No API keys needed for THC +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +}