admin/projectdiscovery-subfinder
1
0
Fork 0
mirror of https://github.com/projectdiscovery/subfinder synced 2026-06-21 14:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(crtsh): drop unused certificate metadata from SQL query (#1773) (#1787)

Browse Source 
The Postgres query inherited the verbatim form crt.sh's web UI uses to
render its certificate browser. subfinder consumes NAME_VALUE only, so
GROUP BY sub.CERTIFICATE on multi-KB DER blobs, the four x509_*
function calls per group, the correlated LEFT JOIN LATERAL into
ct_log_entry, and the trailing ORDER BY ENTRY_TIMESTAMP are all
discarded.

Replacing the WITH/JOIN with a direct DISTINCT NAME_VALUE preserves the
result set (verified parity on iana.org: 24 unique names from both
queries) and lets the LIMIT 10000 clause bound unique subdomains
instead of raw certificate rows. The downstream SplitSeq + Extractor
path is unchanged.

Co-authored-by: Chris (ChrisJr404) <11917633+ChrisJr404@users.noreply.github.com>
This commit is contained in:
ChrisJr404
2026-06-16 09:29:27 -04:00
committed by GitHub
parent 2a8ce2f3e6
commit 292032eef4
1 changed files with 8 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/subscraping/sources/crtsh/crtsh.go
+8 -26
View File
@@ -79,32 +79,14 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio
}
}
query := fmt.Sprintf(`WITH ci AS (
SELECT min(sub.CERTIFICATE_ID) ID,
min(sub.ISSUER_CA_ID) ISSUER_CA_ID,
array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES,
x509_commonName(sub.CERTIFICATE) COMMON_NAME,
x509_notBefore(sub.CERTIFICATE) NOT_BEFORE,
x509_notAfter(sub.CERTIFICATE) NOT_AFTER,
encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER
FROM (SELECT *
FROM certificate_and_identities cai
WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%')
%s
) sub
GROUP BY sub.CERTIFICATE
)
SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE
FROM ci
LEFT JOIN LATERAL (
SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP
FROM ct_log_entry ctle
WHERE ctle.CERTIFICATE_ID = ci.ID
) le ON TRUE,
ca
WHERE ci.ISSUER_CA_ID = ca.ID
ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause)
// We only consume NAME_VALUE downstream, so query for that directly instead
// of joining ct_log_entry / running x509_* parsers on every certificate.
// See https://github.com/projectdiscovery/subfinder/issues/1773.
query := fmt.Sprintf(`SELECT DISTINCT cai.NAME_VALUE
FROM certificate_and_identities cai
WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%')
%s;`, limitClause)
rows, err := db.QueryContext(ctx, query, domain)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}