admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2366 from projectdiscovery/feature/cpe-wordpress-detection

Browse Source 
feat: add passive CPE and WordPress detection
This commit is contained in:
Dogan Can Bakir
2026-01-08 19:56:10 +07:00
committed by GitHub
parent bc2c7a2dd3 81461d35e8
commit 834bbd75f3
8 changed files with 421 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -2
View File
@@ -110,9 +110,11 @@ PROBES:
-title display page title
-bp, -body-preview display first N characters of response body (default 100)
-server, -web-server display server name
-td, -tech-detect display technology in use based on wappalyzer dataset
-td, -tech-detect display technology in use based on wappalyzer dataset
-cff, -custom-fingerprint-file string path to a custom fingerprint file for technology detection
-method display http request method
-cpe display CPE (Common Platform Enumeration) based on awesome-search-queries
-wp, -wordpress display WordPress plugins and themes
-method display http request method
-ws, -websocket display server using websocket
-ip display host ip
-cname display host cname
go.mod
+1
View File
@@ -128,6 +128,7 @@ require (
github.com/pierrec/lz4/v4 v4.1.23 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/gostruct v0.0.2 // indirect
go.sum
+2
View File
@@ -320,6 +320,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193 h1:UCZRqs1BP1wsvhCwQxfIQc7NJcXGBhQvAnEw3awhsng=
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260104120501-961ef30f7193/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.2.17 h1:Ah7KIft60ZiE6etGuX/63HiDJu0C7szhEwYTQugVorU=
runner/cpe.go
+225
View File
@@ -0,0 +1,225 @@
package runner
import (
"encoding/json"
"fmt"
"strings"
awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
)
type CPEInfo struct {
Product string `json:"product,omitempty"`
Vendor string `json:"vendor,omitempty"`
CPE string `json:"cpe,omitempty"`
}
type CPEDetector struct {
titlePatterns map[string][]CPEInfo
bodyPatterns map[string][]CPEInfo
faviconPatterns map[string][]CPEInfo
}
type rawQuery struct {
Name string `json:"name"`
Vendor json.RawMessage `json:"vendor"`
Type string `json:"type"`
Engines []rawEngine `json:"engines"`
}
type rawEngine struct {
Platform string `json:"platform"`
Queries []string `json:"queries"`
}
func NewCPEDetector() (*CPEDetector, error) {
data, err := awesomesearchqueries.GetQueries()
if err != nil {
return nil, fmt.Errorf("failed to load queries: %w", err)
}
var queries []rawQuery
if err := json.Unmarshal(data, &queries); err != nil {
return nil, fmt.Errorf("failed to parse queries: %w", err)
}
detector := &CPEDetector{
titlePatterns: make(map[string][]CPEInfo),
bodyPatterns: make(map[string][]CPEInfo),
faviconPatterns: make(map[string][]CPEInfo),
}
for _, q := range queries {
vendor := parseVendor(q.Vendor)
info := CPEInfo{
Product: q.Name,
Vendor: vendor,
CPE: generateCPE(vendor, q.Name),
}
for _, engine := range q.Engines {
for _, query := range engine.Queries {
detector.extractPattern(query, info)
}
}
}
return detector, nil
}
func parseVendor(raw json.RawMessage) string {
var vendorStr string
if err := json.Unmarshal(raw, &vendorStr); err == nil {
return vendorStr
}
var vendorSlice []string
if err := json.Unmarshal(raw, &vendorSlice); err == nil && len(vendorSlice) > 0 {
return vendorSlice[0]
}
return ""
}
func generateCPE(vendor, product string) string {
if vendor == "" || product == "" {
return ""
}
return fmt.Sprintf("cpe:2.3:a:%s:%s:*:*:*:*:*:*:*:*",
strings.ToLower(strings.ReplaceAll(vendor, " ", "_")),
strings.ToLower(strings.ReplaceAll(product, " ", "_")))
}
func (d *CPEDetector) extractPattern(query string, info CPEInfo) {
query = strings.TrimSpace(query)
titlePrefixes := []string{
"http.title:",
"title=",
"title==",
"intitle:",
"title:",
"title='",
`title="`,
}
for _, prefix := range titlePrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
pattern = strings.TrimPrefix(pattern, prefix[:len(prefix)-1])
if pattern != "" {
pattern = strings.ToLower(pattern)
d.titlePatterns[pattern] = appendUnique(d.titlePatterns[pattern], info)
}
return
}
}
bodyPrefixes := []string{
"http.html:",
"body=",
"body==",
"intext:",
}
for _, prefix := range bodyPrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
if pattern != "" {
pattern = strings.ToLower(pattern)
d.bodyPatterns[pattern] = appendUnique(d.bodyPatterns[pattern], info)
}
return
}
}
faviconPrefixes := []string{
"http.favicon.hash:",
"icon_hash=",
"icon_hash==",
}
for _, prefix := range faviconPrefixes {
if strings.HasPrefix(strings.ToLower(query), strings.ToLower(prefix)) {
pattern := extractQuotedValue(strings.TrimPrefix(query, prefix))
if pattern != "" {
d.faviconPatterns[pattern] = appendUnique(d.faviconPatterns[pattern], info)
}
return
}
}
}
func extractQuotedValue(s string) string {
s = strings.TrimSpace(s)
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
if idx := strings.Index(s, "\" ||"); idx > 0 {
s = s[:idx]
}
if idx := strings.Index(s, "' ||"); idx > 0 {
s = s[:idx]
}
return strings.TrimSpace(s)
}
func appendUnique(slice []CPEInfo, info CPEInfo) []CPEInfo {
for _, existing := range slice {
if existing.Product == info.Product && existing.Vendor == info.Vendor {
return slice
}
}
return append(slice, info)
}
func (d *CPEDetector) Detect(title, body, faviconHash string) []CPEInfo {
seen := make(map[string]bool)
var results []CPEInfo
titleLower := strings.ToLower(title)
bodyLower := strings.ToLower(body)
for pattern, infos := range d.titlePatterns {
if strings.Contains(titleLower, pattern) {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
for pattern, infos := range d.bodyPatterns {
if strings.Contains(bodyLower, pattern) {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
if faviconHash != "" {
if infos, ok := d.faviconPatterns[faviconHash]; ok {
for _, info := range infos {
key := info.Product + "|" + info.Vendor
if !seen[key] {
seen[key] = true
results = append(results, info)
}
}
}
}
return results
}
runner/options.go
+8
View File
@@ -85,6 +85,8 @@ type ScanOptions struct {
NoFallback bool
NoFallbackScheme bool
TechDetect bool
CPEDetect bool
WordPress bool
StoreChain bool
StoreVisionReconClusters bool
MaxResponseBodySizeToSave int
@@ -148,6 +150,8 @@ func (s *ScanOptions) Clone() *ScanOptions {
NoFallback: s.NoFallback,
NoFallbackScheme: s.NoFallbackScheme,
TechDetect: s.TechDetect,
CPEDetect: s.CPEDetect,
WordPress: s.WordPress,
StoreChain: s.StoreChain,
OutputExtractRegex: s.OutputExtractRegex,
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
@@ -256,6 +260,8 @@ type Options struct {
NoFallback bool
NoFallbackScheme bool
TechDetect bool
CPEDetect bool
WordPress bool
CustomFingerprintFile string
TLSGrab bool
protocol string
@@ -387,6 +393,8 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
flagSet.StringVarP(&options.CustomFingerprintFile, "custom-fingerprint-file", "cff", "", "path to a custom fingerprint file for technology detection"),
flagSet.BoolVar(&options.CPEDetect, "cpe", false, "display CPE (Common Platform Enumeration) based on awesome-search-queries"),
flagSet.BoolVarP(&options.WordPress, "wordpress", "wp", false, "display WordPress plugins and themes"),
flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
flagSet.BoolVarP(&options.OutputWebSocket, "websocket", "ws", false, "display server using websocket"),
flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
runner/runner.go
+61
View File
@@ -81,6 +81,8 @@ type Runner struct {
options *Options
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
cpeDetector *CPEDetector
wpDetector *WordPressDetector
scanopts ScanOptions
hm *hybrid.HybridMap
excludeCdn bool
@@ -133,6 +135,20 @@ func New(options *Options) (*Runner, error) {
return nil, errors.Wrap(err, "could not create wappalyzer client")
}
if options.CPEDetect || options.JSONOutput || options.CSVOutput {
runner.cpeDetector, err = NewCPEDetector()
if err != nil {
gologger.Warning().Msgf("Could not create CPE detector: %s", err)
}
}
if options.WordPress || options.JSONOutput || options.CSVOutput {
runner.wpDetector, err = NewWordPressDetector()
if err != nil {
gologger.Warning().Msgf("Could not create WordPress detector: %s", err)
}
}
if options.StoreResponseDir != "" {
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
_ = os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
@@ -297,6 +313,8 @@ func New(options *Options) (*Runner, error) {
scanopts.NoFallback = options.NoFallback
scanopts.NoFallbackScheme = options.NoFallbackScheme
scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload
scanopts.CPEDetect = options.CPEDetect || options.JSONOutput || options.CSVOutput
scanopts.WordPress = options.WordPress || options.JSONOutput || options.CSVOutput
scanopts.StoreChain = options.StoreChain
scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters
scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave
@@ -2311,6 +2329,47 @@ retry:
}
}
var cpeMatches []CPEInfo
if r.cpeDetector != nil {
cpeMatches = r.cpeDetector.Detect(title, string(resp.Data), faviconMMH3)
if len(cpeMatches) > 0 && r.options.CPEDetect {
for _, cpe := range cpeMatches {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Cyan(cpe.CPE).String())
} else {
builder.WriteString(cpe.CPE)
}
builder.WriteRune(']')
}
}
}
var wpInfo *WordPressInfo
if r.wpDetector != nil {
wpInfo = r.wpDetector.Detect(string(resp.Data))
if wpInfo.HasData() && r.options.WordPress {
if len(wpInfo.Plugins) > 0 {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Green("wp-plugins:" + strings.Join(wpInfo.Plugins, ",")).String())
} else {
builder.WriteString("wp-plugins:" + strings.Join(wpInfo.Plugins, ","))
}
builder.WriteRune(']')
}
if len(wpInfo.Themes) > 0 {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Green("wp-themes:" + strings.Join(wpInfo.Themes, ",")).String())
} else {
builder.WriteString("wp-themes:" + strings.Join(wpInfo.Themes, ","))
}
builder.WriteRune(']')
}
}
}
result := Result{
Timestamp: time.Now(),
Request: request,
@@ -2374,6 +2433,8 @@ retry:
RequestRaw: requestDump,
Response: resp,
FaviconData: faviconData,
CPE: cpeMatches,
WordPress: wpInfo,
}
if resp.BodyDomains != nil {
result.Fqdns = resp.BodyDomains.Fqdns
runner/types.go
+2
View File
@@ -102,6 +102,8 @@ type Result struct {
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"`
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" mapstructure:"cpe"`
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" mapstructure:"wordpress"`
}
type Trace struct {
runner/wordpress.go
+118
View File
@@ -0,0 +1,118 @@
package runner
import (
"bufio"
"bytes"
"regexp"
"strings"
awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
)
type WordPressInfo struct {
Plugins []string `json:"plugins,omitempty"`
Themes []string `json:"themes,omitempty"`
}
type WordPressDetector struct {
knownPlugins map[string]struct{}
knownThemes map[string]struct{}
pluginRegex *regexp.Regexp
themeRegex *regexp.Regexp
}
func NewWordPressDetector() (*WordPressDetector, error) {
detector := &WordPressDetector{
knownPlugins: make(map[string]struct{}),
knownThemes: make(map[string]struct{}),
}
var err error
detector.pluginRegex, err = regexp.Compile(`/wp-content/plugins/([a-zA-Z0-9_-]+)/`)
if err != nil {
return nil, err
}
detector.themeRegex, err = regexp.Compile(`/wp-content/themes/([a-zA-Z0-9_-]+)/`)
if err != nil {
return nil, err
}
pluginsData, err := awesomesearchqueries.GetWordPressPlugins()
if err != nil {
return nil, err
}
if err := detector.loadList(pluginsData, detector.knownPlugins); err != nil {
return nil, err
}
themesData, err := awesomesearchqueries.GetWordPressThemes()
if err != nil {
return nil, err
}
if err := detector.loadList(themesData, detector.knownThemes); err != nil {
return nil, err
}
return detector, nil
}
func (d *WordPressDetector) loadList(data []byte, target map[string]struct{}) error {
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
target[line] = struct{}{}
}
}
return scanner.Err()
}
func (d *WordPressDetector) Detect(body string) *WordPressInfo {
if body == "" {
return nil
}
info := &WordPressInfo{}
seenPlugins := make(map[string]struct{})
seenThemes := make(map[string]struct{})
if matches := d.pluginRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 {
for _, match := range matches {
if len(match) > 1 {
plugin := match[1]
if _, seen := seenPlugins[plugin]; !seen {
if _, known := d.knownPlugins[plugin]; known {
info.Plugins = append(info.Plugins, plugin)
seenPlugins[plugin] = struct{}{}
}
}
}
}
}
if matches := d.themeRegex.FindAllStringSubmatch(body, -1); len(matches) > 0 {
for _, match := range matches {
if len(match) > 1 {
theme := match[1]
if _, seen := seenThemes[theme]; !seen {
if _, known := d.knownThemes[theme]; known {
info.Themes = append(info.Themes, theme)
seenThemes[theme] = struct{}{}
}
}
}
}
}
if len(info.Plugins) == 0 && len(info.Themes) == 0 {
return nil
}
return info
}
func (w *WordPressInfo) HasData() bool {
return w != nil && (len(w.Plugins) > 0 || len(w.Themes) > 0)
}