new source dnsrepo (#501)

* new source dnsrepo

* added api keys for dnsrepo

* source added to test case

* integration test set up and test case for dnsrepo

* lint error fix

* misc update

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/build-test.yml

@@ -28,12 +28,12 @@ jobs:
         run: go test ./...
         working-directory: v2/
 
-      # Todo
-      # - name: Integration Tests
-      #   env:
-      #     GH_ACTION: true
-      #   run: bash run.sh
-      #   working-directory: integration_tests/
+      - name: Integration Tests
+        env:
+          GH_ACTION: true
+          DNSREPO_API_KEY: ${{secrets.DNSREPO_API}}
+        run: bash run.sh
+        working-directory: v2/cmd/integration-test/
 
       - name: Race Condition Tests
         run: go build -race ./...

v2/cmd/integration-test/integration-test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/testutils"
+)
+
+var (
+	debug        = os.Getenv("DEBUG") == "true"
+	githubAction = os.Getenv("GH_ACTION") == "true"
+	customTests  = os.Getenv("TESTS")
+
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+
+	sourceTests = map[string]testutils.TestCase{
+		"dnsrepo": dnsrepoTestcases{},
+	}
+)
+
+func main() {
+	failedTestCases := runTests(toMap(toSlice(customTests)))
+
+	if len(failedTestCases) > 0 {
+		if githubAction {
+			debug = true
+			fmt.Println("::group::Failed integration tests in debug mode")
+			_ = runTests(failedTestCases)
+			fmt.Println("::endgroup::")
+		}
+		os.Exit(1)
+	}
+}
+
+func runTests(customTestCases map[string]struct{}) map[string]struct{} {
+	failedTestCases := map[string]struct{}{}
+
+	for source, testCase := range sourceTests {
+		if len(customTestCases) == 0 {
+			fmt.Printf("Running test cases for %q source\n", aurora.Blue(source))
+		}
+		if err, failedTemplatePath := execute(source, testCase); err != nil {
+			failedTestCases[failedTemplatePath] = struct{}{}
+		}
+	}
+	return failedTestCases
+}
+
+func execute(source string, testCase testutils.TestCase) (error, string) {
+	if err := testCase.Execute(); err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, source, err)
+		return err, source
+	}
+
+	fmt.Printf("%s Test \"%s\" passed!\n", success, source)
+	return nil, ""
+}
+
+func expectResultsGreaterThanCount(results []string, expectedNumber int) error {
+	if len(results) > expectedNumber {
+		return nil
+	}
+	return fmt.Errorf("incorrect number of results: expected a result greater than %d,but got %d", expectedNumber, len(results))
+}
+func toSlice(value string) []string {
+	if strings.TrimSpace(value) == "" {
+		return []string{}
+	}
+
+	return strings.Split(value, ",")
+}
+
+func toMap(slice []string) map[string]struct{} {
+	result := make(map[string]struct{}, len(slice))
+	for _, value := range slice {
+		if _, ok := result[value]; !ok {
+			result[value] = struct{}{}
+		}
+	}
+	return result
+}

v2/cmd/integration-test/run.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo "::task~> Clean up & Build binaries files"
+rm integration-test subfinder 2>/dev/null
+cd ../subfinder
+go build
+mv subfinder ../integration-test/subfinder
+cd ../integration-test
+go build
+echo "::done::"
+echo "::task~> Run integration test"
+./integration-test
+echo "::done::"
+if [ $? -eq 0 ]
+then
+  exit 0
+else
+  exit 1
+fi

v2/cmd/integration-test/source-test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/testutils"
+)
+
+type dnsrepoTestcases struct{}
+
+func (h dnsrepoTestcases) Execute() error {
+	token := os.Getenv("DNSREPO_API_KEY")
+	if token == "" {
+		return errors.New("missing dns repo api key")
+	}
+	dnsToken := fmt.Sprintf(`dnsrepo: [%s]`, token)
+	file, err := os.CreateTemp("", "provider.yaml")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(file.Name())
+	_, err = file.WriteString(dnsToken)
+	if err != nil {
+		return err
+	}
+	results, err := testutils.RunSubfinderAndGetResults(debug, "hackerone.com", "-s", "dnsrepo", "-provider-config", file.Name())
+	if err != nil {
+		return err
+	}
+	return expectResultsGreaterThanCount(results, 0)
+}

v2/go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/mholt/archiver v3.1.1+incompatible // indirect
-	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.20 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect
@@ -49,7 +49,7 @@ require (
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
+	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/miekg/dns v1.1.46 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -58,6 +58,6 @@ require (
 	github.com/projectdiscovery/goflags v0.0.9
 	github.com/projectdiscovery/retryabledns v1.0.13 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 // indirect
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 )

v2/go.sum

@@ -54,6 +54,8 @@ github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1w
 github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
 github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo=
 github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
+github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
+github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
 github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -156,6 +158,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
@@ -171,6 +175,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

v2/pkg/passive/sources.go

@@ -45,6 +45,8 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
+
 )
 
 var AllSources = [...]subscraping.Source{
@@ -85,6 +87,7 @@ var AllSources = [...]subscraping.Source{
 	&whoisxmlapi.Source{},
 	&zoomeye.Source{},
 	&zoomeyeapi.Source{},
+	&dnsrepo.Source{},
 }
 
 var NameSourceMap = make(map[string]subscraping.Source, len(AllSources))

v2/pkg/passive/sources_test.go

@@ -26,6 +26,7 @@ var (
 		"crtsh",
 		"dnsdumpster",
 		"dnsdb",
+		"dnsrepo",
 		"fofa",
 		"fullhunt",
 		"github",
@@ -62,6 +63,7 @@ var (
 		"chinaz",
 		"crtsh",
 		"dnsdumpster",
+		"dnsrepo",
 		"fofa",
 		"fullhunt",
 		"hackertarget",

v2/pkg/runner/options.go

@@ -166,6 +166,12 @@ func ParseOptions() *Options {
 		os.Exit(0)
 	}
 
+	options.preProcessOptions()
+
+	if !options.Silent {
+		showBanner()
+	}
+
 	// Check if the application loading with any provider configuration, then take it
 	// Otherwise load the default provider config
 	if fileutil.FileExists(options.ProviderConfig) {
@@ -180,12 +186,6 @@ func ParseOptions() *Options {
 		os.Exit(0)
 	}
 
-	options.preProcessOptions()
-
-	if !options.Silent {
-		showBanner()
-	}
-
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
 	err = options.validateOptions()

v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go

@@ -0,0 +1,79 @@
+package dnsrepo
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+// Source is the passive scraping agent
+type Source struct {
+	apiKeys []string
+}
+
+type DnsRepoResponse []struct {
+	Domain string
+}
+
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+
+	go func() {
+		defer close(results)
+
+		randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
+		if randomApiKey == "" {
+			return
+		}
+		resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain))
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		responseData, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		resp.Body.Close()
+		var result DnsRepoResponse
+		err = json.Unmarshal(responseData, &result)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+		for _, sub := range result {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}
+		}
+
+	}()
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "dnsrepo"
+}
+
+func (s *Source) IsDefault() bool {
+	return true
+}
+
+func (s *Source) HasRecursiveSupport() bool {
+	return false
+}
+
+func (s *Source) NeedsKey() bool {
+	return true
+}
+
+func (s *Source) AddApiKeys(keys []string) {
+	s.apiKeys = keys
+}

v2/pkg/testutils/integration.go

@@ -0,0 +1,42 @@
+package testutils
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) {
+	cmd := exec.Command("bash", "-c")
+	cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ")
+	cmdLine += strings.Join(extra, " ")
+	cmd.Args = append(cmd.Args, cmdLine)
+	if debug {
+		cmd.Args = append(cmd.Args, "-v")
+		cmd.Stderr = os.Stderr
+		fmt.Println(cmd.String())
+	} else {
+		cmd.Args = append(cmd.Args, "-silent")
+	}
+	data, err := cmd.Output()
+	if debug {
+		fmt.Println(string(data))
+	}
+	if err != nil {
+		return nil, err
+	}
+	var parts []string
+	items := strings.Split(string(data), "\n")
+	for _, i := range items {
+		if i != "" {
+			parts = append(parts, i)
+		}
+	}
+	return parts, nil
+}
+
+// TestCase is a single integration test case
+type TestCase interface {
+	Execute() error
+}