From 292032eef4ad0d2a18f3fac950ec6aff1b3a2edb Mon Sep 17 00:00:00 2001 From: ChrisJr404 Date: Tue, 16 Jun 2026 09:29:27 -0400 Subject: [PATCH] fix(crtsh): drop unused certificate metadata from SQL query (#1773) (#1787) The Postgres query inherited the verbatim form crt.sh's web UI uses to render its certificate browser. subfinder consumes NAME_VALUE only, so GROUP BY sub.CERTIFICATE on multi-KB DER blobs, the four x509_* function calls per group, the correlated LEFT JOIN LATERAL into ct_log_entry, and the trailing ORDER BY ENTRY_TIMESTAMP are all discarded. Replacing the WITH/JOIN with a direct DISTINCT NAME_VALUE preserves the result set (verified parity on iana.org: 24 unique names from both queries) and lets the LIMIT 10000 clause bound unique subdomains instead of raw certificate rows. The downstream SplitSeq + Extractor path is unchanged. Co-authored-by: Chris (ChrisJr404) <11917633+ChrisJr404@users.noreply.github.com> --- pkg/subscraping/sources/crtsh/crtsh.go | 34 ++++++-------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index a22b5bdc..578e2080 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -79,32 +79,14 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } } - query := fmt.Sprintf(`WITH ci AS ( - SELECT min(sub.CERTIFICATE_ID) ID, - min(sub.ISSUER_CA_ID) ISSUER_CA_ID, - array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES, - x509_commonName(sub.CERTIFICATE) COMMON_NAME, - x509_notBefore(sub.CERTIFICATE) NOT_BEFORE, - x509_notAfter(sub.CERTIFICATE) NOT_AFTER, - encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER - FROM (SELECT * - FROM certificate_and_identities cai - WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE) - AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%') - %s - ) sub - GROUP BY sub.CERTIFICATE - ) - SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE - FROM ci - LEFT JOIN LATERAL ( - SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP - FROM ct_log_entry ctle - WHERE ctle.CERTIFICATE_ID = ci.ID - ) le ON TRUE, - ca - WHERE ci.ISSUER_CA_ID = ca.ID - ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause) + // We only consume NAME_VALUE downstream, so query for that directly instead + // of joining ct_log_entry / running x509_* parsers on every certificate. + // See https://github.com/projectdiscovery/subfinder/issues/1773. + query := fmt.Sprintf(`SELECT DISTINCT cai.NAME_VALUE + FROM certificate_and_identities cai + WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE) + AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%') + %s;`, limitClause) rows, err := db.QueryContext(ctx, query, domain) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}