Added pugrecon.com as a subdomains source (#1585)

v2/pkg/passive/sources.go

@@ -20,6 +20,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
@@ -34,6 +35,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs"
@@ -47,7 +49,6 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
-	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
@@ -77,6 +78,7 @@ var AllSources = [...]subscraping.Source{
 	&netlas.Source{},
 	&leakix.Source{},
 	&quake.Source{},
+	&pugrecon.Source{},
 	&rapiddns.Source{},
 	&redhuntlabs.Source{},
 	// &riddler.Source{}, // failing due to cloudfront protection

v2/pkg/passive/sources_test.go

@@ -34,6 +34,7 @@ var (
 		"intelx",
 		"netlas",
 		"quake",
+		"pugrecon",
 		"rapiddns",
 		"redhuntlabs",
 		// "riddler", // failing due to cloudfront protection

v2/pkg/runner/options.go

@@ -244,6 +244,7 @@ func (options *Options) preProcessDomains() {
 var defaultRateLimits = []string{
 	"github=30/m",
 	"fullhunt=60/m",
+	"pugrecon=10/s",
 	fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)),
 	"securitytrails=1/s",
 	"shodan=1/s",

v2/pkg/subscraping/sources/pugrecon/pugrecon.go

@@ -0,0 +1,151 @@
+// Package pugrecon logic
+package pugrecon
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+// pugreconResult stores a single result from the pugrecon API
+type pugreconResult struct {
+	Name string `json:"name"`
+}
+
+// pugreconAPIResponse stores the response from the pugrecon API
+type pugreconAPIResponse struct {
+	Results        []pugreconResult `json:"results"`
+	QuotaRemaining int              `json:"quota_remaining"`
+	Limited        bool             `json:"limited"`
+	TotalResults   int              `json:"total_results"`
+	Message        string           `json:"message"`
+}
+
+// Source is the passive scraping agent
+type Source struct {
+	apiKeys   []string
+	timeTaken time.Duration
+	errors    int
+	results   int
+	skipped   bool
+}
+
+// Run function returns all subdomains found with the service
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+	s.errors = 0
+	s.results = 0
+
+	go func() {
+		defer func(startTime time.Time) {
+			s.timeTaken = time.Since(startTime)
+			close(results)
+		}(time.Now())
+
+		randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
+		if randomApiKey == "" {
+			s.skipped = true
+			return
+		}
+
+		// Prepare POST request data
+		postData := map[string]string{"domain_name": domain}
+		bodyBytes, err := json.Marshal(postData)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)}
+			s.errors++
+			return
+		}
+		bodyReader := bytes.NewReader(bodyBytes)
+
+		// Prepare headers
+		headers := map[string]string{
+			"Authorization": "Bearer " + randomApiKey,
+			"Content-Type":  "application/json",
+			"Accept":        "application/json",
+		}
+
+		apiURL := "https://pugrecon.com/api/v1/domains"
+		resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode)
+			// Attempt to read error message from body if possible
+			var apiResp pugreconAPIResponse
+			if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" {
+				errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message)
+			}
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(errorMsg)}
+			s.errors++
+			return
+		}
+
+		var response pugreconAPIResponse
+		err = json.NewDecoder(resp.Body).Decode(&response)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			return
+		}
+
+		if response.Message != "" && !response.Limited { // Handle potential non-error messages, except rate limit info
+			// Log or handle message if needed, but don't treat as hard error unless necessary
+		}
+
+		for _, subdomain := range response.Results {
+			results <- subscraping.Result{
+				Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name,
+			}
+			s.results++
+		}
+	}()
+
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "pugrecon"
+}
+
+// IsDefault returns false as this is not a default source.
+func (s *Source) IsDefault() bool {
+	return false
+}
+
+// HasRecursiveSupport returns false as this source does not support recursive searches.
+func (s *Source) HasRecursiveSupport() bool {
+	return false
+}
+
+// NeedsKey returns true as this source requires an API key.
+func (s *Source) NeedsKey() bool {
+	return true
+}
+
+// AddApiKeys adds the API keys for the source.
+func (s *Source) AddApiKeys(keys []string) {
+	s.apiKeys = keys
+}
+
+// Statistics returns the statistics for the source.
+func (s *Source) Statistics() subscraping.Statistics {
+	return subscraping.Statistics{
+		Errors:    s.errors,
+		Results:   s.results,
+		TimeTaken: s.timeTaken,
+		Skipped:   s.skipped,
+	}
+}