allow comments in input file (#1346)

* allow comments in input file

* use str util func

* fix lint

* lint fix

v2/go.mod

@@ -120,7 +120,7 @@ require (
 	github.com/miekg/dns v1.1.56 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pkg/errors v0.9.1
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/goflags v0.1.63
 	github.com/projectdiscovery/retryabledns v1.0.74 // indirect

v2/pkg/passive/sources.go

@@ -1,7 +1,6 @@
 package passive
 
 import (
-	"fmt"
 	"strings"
 
 	"golang.org/x/exp/maps"
@@ -157,7 +156,7 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup
 		}
 	}
 
-	gologger.Debug().Msgf(fmt.Sprintf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", ")))
+	gologger.Debug().Msgf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", "))
 
 	for _, currentSource := range sources {
 		if warning, ok := sourceWarnings.Get(strings.ToLower(currentSource.Name())); ok {

v2/pkg/runner/options.go

@@ -215,7 +215,7 @@ func (options *Options) loadProvidersFrom(location string) {
 
 	// We skip bailing out if file doesn't exist because we'll create it
 	// at the end of options parsing from default via goflags.
-	if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(os.ErrNotExist, err)) {
+	if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(err, os.ErrNotExist)) {
 		gologger.Error().Msgf("Could not read providers from %s: %s\n", location, err)
 	}
 }
@@ -237,7 +237,7 @@ func listSources(options *Options) {
 
 func (options *Options) preProcessDomains() {
 	for i, domain := range options.Domain {
-		options.Domain[i], _ = sanitize(domain)
+		options.Domain[i] = preprocessDomain(domain)
 	}
 }
 

v2/pkg/runner/runner.go

@@ -11,8 +11,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"github.com/projectdiscovery/gologger"
 	contextutil "github.com/projectdiscovery/utils/context"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -116,12 +114,13 @@ func (r *Runner) EnumerateMultipleDomains(reader io.Reader, writers []io.Writer)
 // EnumerateMultipleDomainsWithCtx enumerates subdomains for multiple domains
 // We keep enumerating subdomains for a given domain until we reach an error
 func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.Reader, writers []io.Writer) error {
+	var err error
 	scanner := bufio.NewScanner(reader)
 	ip, _ := regexp.Compile(`^([0-9\.]+$)`)
 	for scanner.Scan() {
-		domain, err := normalizeLowercase(scanner.Text())
-		isIp := ip.MatchString(domain)
-		if errors.Is(err, ErrEmptyInput) || (r.options.ExcludeIps && isIp) {
+		domain := preprocessDomain(scanner.Text())
+
+		if domain == "" || (r.options.ExcludeIps && ip.MatchString(domain)) {
 			continue
 		}
 

v2/pkg/runner/stats.go

@@ -30,14 +30,12 @@ func printStatistics(stats map[string]subscraping.Statistics) {
 
 	if len(lines) > 0 {
 		gologger.Print().Msgf("\n Source               Duration      Results     Errors\n%s\n", strings.Repeat("─", 56))
-		gologger.Print().Msgf(strings.Join(lines, "\n"))
-		gologger.Print().Msgf("\n")
+		gologger.Print().Msgf("%s\n", strings.Join(lines, "\n"))
 	}
 
 	if len(skipped) > 0 {
 		gologger.Print().Msgf("\n The following sources were included but skipped...\n\n")
-		gologger.Print().Msgf(strings.Join(skipped, "\n"))
-		gologger.Print().Msgf("\n\n")
+		gologger.Print().Msgf("%s\n\n", strings.Join(skipped, "\n"))
 	}
 }
 

v2/pkg/runner/util.go

@@ -1,15 +1,8 @@
 package runner
 
 import (
-	"strings"
-
-	"github.com/pkg/errors"
-
 	fileutil "github.com/projectdiscovery/utils/file"
-)
-
-var (
-	ErrEmptyInput = errors.New("empty data")
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 func loadFromFile(file string) ([]string, error) {
@@ -19,9 +12,8 @@ func loadFromFile(file string) ([]string, error) {
 	}
 	var items []string
 	for item := range chanItems {
-		var err error
-		item, err = sanitize(item)
-		if errors.Is(err, ErrEmptyInput) {
+		item = preprocessDomain(item)
+		if item == "" {
 			continue
 		}
 		items = append(items, item)
@@ -29,15 +21,12 @@ func loadFromFile(file string) ([]string, error) {
 	return items, nil
 }
 
-func sanitize(data string) (string, error) {
-	data = strings.Trim(data, "\n\t\"'` ")
-	if data == "" {
-		return "", ErrEmptyInput
-	}
-	return data, nil
-}
-
-func normalizeLowercase(s string) (string, error) {
-	data, err := sanitize(s)
-	return strings.ToLower(data), err
+func preprocessDomain(s string) string {
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimCutset:    "\n\t\"'` ",
+			Lowercase:     true,
+		},
+	)
 }

v2/pkg/subscraping/sources/binaryedge/binaryedge.go

@@ -119,7 +119,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba
 
 	// Check error messages
 	if response.Message != "" && response.Status != nil {
-		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)}
+		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)}
 		s.errors++
 		return
 	}