Merge pull request #1723 from recepgunes1/fix/timeout-flag-not-working

Fix/timeout flag not working

pkg/subscraping/agent.go

@@ -46,7 +46,7 @@ func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLi
 		Timeout:   time.Duration(timeout) * time.Second,
 	}
 
-	session := &Session{Client: client}
+	session := &Session{Client: client, Timeout: timeout}
 
 	// Initiate rate limit instance
 	session.MultiRateLimiter = multiRateLimiter

pkg/subscraping/sources/crtsh/crtsh.go

@@ -56,7 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 }
 
 func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int {
-	db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes")
+	// connect_timeout: limits connection establishment time (in seconds)
+	// statement_timeout: limits query execution time (in milliseconds)
+	connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000)
+	db, err := sql.Open("postgres", connStr)
 	if err != nil {
 		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 		s.errors++

pkg/subscraping/types.go

@@ -89,6 +89,8 @@ type Session struct {
 	Client *http.Client
 	// Rate limit instance
 	MultiRateLimiter *ratelimit.MultiLimiter
+	// Timeout is the timeout in seconds for requests
+	Timeout int
 }
 
 // Result is a result structure returned by a source