Merge pull request #2344 from ayuxsec/feat-add-md-output-support

Add Markdown output support
This commit is contained in:
Dogan Can Bakir
2026-03-05 14:16:46 +03:00
committed by GitHub
4 changed files with 218 additions and 82 deletions
+96
View File
@@ -0,0 +1,96 @@
package runner
import (
"fmt"
"reflect"
"strings"
)
func (r Result) MarkdownHeader() string { //nolint
var headers []string
t := reflect.TypeOf(r)
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
tag := field.Tag.Get("md")
if tag == "" || tag == "-" {
continue
}
headers = append(headers, tag)
}
var b strings.Builder
b.WriteString("|")
for _, h := range headers {
fmt.Fprintf(&b, " %s |", h)
}
b.WriteString("\n")
b.WriteString("|")
for range headers {
b.WriteString("---|")
}
b.WriteString("\n")
return b.String()
}
func (r Result) MarkdownRow(scanopts *ScanOptions) string { //nolint
var values []string
v := reflect.ValueOf(r)
t := v.Type()
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
tag := field.Tag.Get("md")
if tag == "" || tag == "-" {
continue
}
fieldValue := v.Field(i)
values = append(values, formatMarkdownValue(fieldValue))
}
var b strings.Builder
b.WriteString("|")
for _, val := range values {
fmt.Fprintf(&b, " %s |", val)
}
b.WriteString("\n")
return b.String()
}
func formatMarkdownValue(v reflect.Value) string {
switch v.Kind() {
case reflect.String:
return escapeMarkdown(v.String())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return fmt.Sprintf("%d", v.Int())
case reflect.Bool:
return fmt.Sprintf("%t", v.Bool())
case reflect.Slice:
if v.Len() == 0 {
return ""
}
var items []string
for i := 0; i < v.Len(); i++ {
items = append(items, fmt.Sprintf("%v", v.Index(i).Interface()))
}
return escapeMarkdown(strings.Join(items, ", "))
default:
if v.CanInterface() {
return escapeMarkdown(fmt.Sprintf("%v", v.Interface()))
}
return ""
}
}
func escapeMarkdown(s string) string {
replacer := strings.NewReplacer(
"|", "\\|",
"\n", " ",
)
return strings.TrimSpace(replacer.Replace(s))
}
+2
View File
@@ -230,6 +230,7 @@ type Options struct {
RespectHSTS bool
StoreResponse bool
JSONOutput bool
MarkDownOutput bool
CSVOutput bool
CSVOutputEncoding string
PdcpAuth string
@@ -503,6 +504,7 @@ func ParseOptions() *Options {
flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"),
flagSet.BoolVarP(&options.JSONOutput, "json", "j", false, "store output in JSONL(ines) format"),
flagSet.BoolVarP(&options.MarkDownOutput, "markdown", "md", false, "store output in Markdown table format"),
flagSet.BoolVarP(&options.ResponseHeadersInStdout, "include-response-header", "irh", false, "include http response (headers) in JSON output (-json only)"),
flagSet.BoolVarP(&options.ResponseInStdout, "include-response", "irr", false, "include http request/response (headers + body) in JSON output (-json only)"),
flagSet.BoolVarP(&options.Base64ResponseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"),
+48 -10
View File
@@ -990,7 +990,8 @@ func (r *Runner) RunEnumeration() {
}
}()
var plainFile, jsonFile, csvFile, indexFile, indexScreenshotFile *os.File
var plainFile, jsonFile, csvFile, mdFile, indexFile, indexScreenshotFile *os.File
markdownHeaderWritten := false // guard to prevent writing the header multiple times
if r.options.Output != "" && r.options.OutputAll {
plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
@@ -1005,11 +1006,15 @@ func (r *Runner) RunEnumeration() {
defer func() {
_ = csvFile.Close()
}()
mdFile = openOrCreateFile(r.options.Resume, r.options.Output+".md")
defer func() {
_ = mdFile.Close()
}()
}
jsonOrCsv := (r.options.JSONOutput || r.options.CSVOutput)
jsonAndCsv := (r.options.JSONOutput && r.options.CSVOutput)
if r.options.Output != "" && plainFile == nil && !jsonOrCsv {
jsonOrCsvOrMD := (r.options.JSONOutput || r.options.CSVOutput || r.options.MarkDownOutput)
jsonAndCsvAndMD := (r.options.JSONOutput && r.options.CSVOutput && r.options.MarkDownOutput)
if r.options.Output != "" && plainFile == nil && !jsonOrCsvOrMD {
plainFile = openOrCreateFile(r.options.Resume, r.options.Output)
defer func() {
_ = plainFile.Close()
@@ -1018,7 +1023,7 @@ func (r *Runner) RunEnumeration() {
if r.options.Output != "" && r.options.JSONOutput && jsonFile == nil {
ext := ""
if jsonAndCsv {
if jsonAndCsvAndMD {
ext = ".json"
}
jsonFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
@@ -1029,7 +1034,7 @@ func (r *Runner) RunEnumeration() {
if r.options.Output != "" && r.options.CSVOutput && csvFile == nil {
ext := ""
if jsonAndCsv {
if jsonAndCsvAndMD {
ext = ".csv"
}
csvFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
@@ -1038,6 +1043,17 @@ func (r *Runner) RunEnumeration() {
}()
}
if r.options.Output != "" && r.options.MarkDownOutput && mdFile == nil {
ext := ""
if jsonAndCsvAndMD {
ext = ".md"
}
mdFile = openOrCreateFile(r.options.Resume, r.options.Output+ext)
defer func() {
_ = mdFile.Close()
}()
}
if r.options.CSVOutput {
outEncoding := strings.ToLower(r.options.CSVOutputEncoding)
switch outEncoding {
@@ -1052,7 +1068,7 @@ func (r *Runner) RunEnumeration() {
gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding)
}
headers := Result{}.CSVHeader()
if !r.options.OutputAll && !jsonAndCsv {
if !r.options.OutputAll && !jsonAndCsvAndMD {
gologger.Silent().Msgf("%s\n", headers)
}
@@ -1273,7 +1289,7 @@ func (r *Runner) RunEnumeration() {
}
}
if !r.options.DisableStdout && (!jsonOrCsv || jsonAndCsv || r.options.OutputAll) {
if !r.options.DisableStdout && (!jsonOrCsvOrMD || jsonAndCsvAndMD || r.options.OutputAll) {
gologger.Silent().Msgf("%s\n", resp.str)
}
@@ -1384,7 +1400,7 @@ func (r *Runner) RunEnumeration() {
if r.options.JSONOutput {
row := resp.JSON(&r.scanopts)
if !r.options.OutputAll && !jsonAndCsv {
if !r.options.OutputAll && !jsonAndCsvAndMD {
gologger.Silent().Msgf("%s\n", row)
}
@@ -1397,7 +1413,7 @@ func (r *Runner) RunEnumeration() {
if r.options.CSVOutput {
row := resp.CSVRow(&r.scanopts)
if !r.options.OutputAll && !jsonAndCsv {
if !r.options.OutputAll && !jsonAndCsvAndMD {
gologger.Silent().Msgf("%s\n", row)
}
@@ -1407,6 +1423,28 @@ func (r *Runner) RunEnumeration() {
}
}
if r.options.MarkDownOutput || r.options.OutputAll {
if !markdownHeaderWritten {
header := resp.MarkdownHeader()
if !r.options.OutputAll {
gologger.Silent().Msgf("%s", header)
}
if mdFile != nil {
_, _ = mdFile.WriteString(header)
}
markdownHeaderWritten = true
}
row := resp.MarkdownRow(&r.scanopts)
if !r.options.OutputAll {
gologger.Silent().Msgf("%s", row)
}
if mdFile != nil {
_, _ = mdFile.WriteString(row)
}
}
for _, nextStep := range nextSteps {
nextStep <- resp
}
+72 -72
View File
@@ -33,78 +33,78 @@ func (o AsnResponse) String() string {
// Result of a scan
type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
LinkRequest []NetworkRequest `json:"link_request,omitempty" csv:"link_request" mapstructure:"link_request"`
ASN *AsnResponse `json:"asn,omitempty" csv:"-" mapstructure:"asn"`
Err error `json:"-" csv:"-" mapstructure:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"-" mapstructure:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"-" mapstructure:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"-" mapstructure:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
Port string `json:"port,omitempty" csv:"port" mapstructure:"port"`
Raw string `json:"-" csv:"-" mapstructure:"-"`
URL string `json:"url,omitempty" csv:"url" mapstructure:"url"`
Input string `json:"input,omitempty" csv:"input" mapstructure:"input"`
Location string `json:"location,omitempty" csv:"location" mapstructure:"location"`
Title string `json:"title,omitempty" csv:"title" mapstructure:"title"`
str string `json:"-" csv:"-" mapstructure:"-"`
Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
Error string `json:"error,omitempty" csv:"error" mapstructure:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"-" mapstructure:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
Method string `json:"method,omitempty" csv:"method" mapstructure:"method"`
Host string `json:"host,omitempty" csv:"host" mapstructure:"host"`
HostIP string `json:"host_ip,omitempty" csv:"host_ip" mapstructure:"host_ip"`
Path string `json:"path,omitempty" csv:"path" mapstructure:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" mapstructure:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"`
Request string `json:"request,omitempty" csv:"-" mapstructure:"request"`
ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a" mapstructure:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"-" mapstructure:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"-" mapstructure:"chain"`
Words int `json:"words" csv:"words" mapstructure:"words"`
Lines int `json:"lines" csv:"lines" mapstructure:"lines"`
StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"`
ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"`
Failed bool `json:"failed" csv:"failed" mapstructure:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"-" mapstructure:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn" mapstructure:"body_fqdn"`
Domains []string `json:"body_domains,omitempty" csv:"body_domains" mapstructure:"body_domains"`
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"`
FileNameHash string `json:"-" csv:"-" mapstructure:"-"`
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" mapstructure:"cpe"`
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" mapstructure:"wordpress"`
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" md:"timestamp" mapstructure:"timestamp"`
LinkRequest []NetworkRequest `json:"link_request,omitempty" csv:"link_request" md:"link_request" mapstructure:"link_request"`
ASN *AsnResponse `json:"asn,omitempty" csv:"-" md:"-" mapstructure:"asn"`
Err error `json:"-" csv:"-" md:"-" mapstructure:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"-" md:"-" mapstructure:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"-" md:"-" mapstructure:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"-" md:"-" mapstructure:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" md:"extract_regex" mapstructure:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" md:"cdn_name" mapstructure:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" md:"cdn_type" mapstructure:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni" md:"sni" mapstructure:"sni"`
Port string `json:"port,omitempty" csv:"port" md:"port" mapstructure:"port"`
Raw string `json:"-" csv:"-" md:"-" mapstructure:"-"`
URL string `json:"url,omitempty" csv:"url" md:"url" mapstructure:"url"`
Input string `json:"input,omitempty" csv:"input" md:"input" mapstructure:"input"`
Location string `json:"location,omitempty" csv:"location" md:"location" mapstructure:"location"`
Title string `json:"title,omitempty" csv:"title" md:"title" mapstructure:"title"`
str string `json:"-" csv:"-" md:"-" mapstructure:"-"`
Scheme string `json:"scheme,omitempty" csv:"scheme" md:"scheme" mapstructure:"scheme"`
Error string `json:"error,omitempty" csv:"error" md:"error" mapstructure:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver" md:"webserver" mapstructure:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"-" md:"-" mapstructure:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" md:"body_preview" mapstructure:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type" md:"content_type" mapstructure:"content_type"`
Method string `json:"method,omitempty" csv:"method" md:"method" mapstructure:"method"`
Host string `json:"host,omitempty" csv:"host" md:"host" mapstructure:"host"`
HostIP string `json:"host_ip,omitempty" csv:"host_ip" md:"host_ip" mapstructure:"host_ip"`
Path string `json:"path,omitempty" csv:"path" md:"path" mapstructure:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" md:"favicon" mapstructure:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" md:"favicon_md5" mapstructure:"favicon_md5"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" md:"favicon_path" mapstructure:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" md:"favicon_url" mapstructure:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url" md:"final_url" mapstructure:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" md:"-" mapstructure:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"-" md:"-" mapstructure:"raw_header"`
Request string `json:"request,omitempty" csv:"-" md:"-" mapstructure:"request"`
ResponseTime string `json:"time,omitempty" csv:"time" md:"time" mapstructure:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" md:"jarm_hash" mapstructure:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" md:"chain_status_codes" mapstructure:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a" md:"a" mapstructure:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" md:"aaaa" mapstructure:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname" md:"cname" mapstructure:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech" md:"tech" mapstructure:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"-" md:"-" mapstructure:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"-" md:"-" mapstructure:"chain"`
Words int `json:"words" csv:"words" md:"words" mapstructure:"words"`
Lines int `json:"lines" csv:"lines" md:"lines" mapstructure:"lines"`
StatusCode int `json:"status_code" csv:"status_code" md:"status_code" mapstructure:"status_code"`
ContentLength int `json:"content_length" csv:"content_length" md:"content_length" mapstructure:"content_length"`
Failed bool `json:"failed" csv:"failed" md:"failed" mapstructure:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost" md:"vhost" mapstructure:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket" md:"websocket" mapstructure:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn" md:"cdn" mapstructure:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2" md:"http2" mapstructure:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" md:"pipeline" mapstructure:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" md:"headless_body" mapstructure:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" md:"screenshot_bytes" mapstructure:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" md:"stored_response_path" mapstructure:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" md:"screenshot_path" mapstructure:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" md:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"-" md:"-" mapstructure:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" md:"resolvers" mapstructure:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty" csv:"body_fqdn" md:"body_fqdn" mapstructure:"body_fqdn"`
Domains []string `json:"body_domains,omitempty" csv:"body_domains" md:"body_domains" mapstructure:"body_domains"`
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" md:"-" mapstructure:"-"`
RequestRaw []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" md:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" md:"-" mapstructure:"-"`
Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" md:"-" mapstructure:"trace"`
FileNameHash string `json:"-" csv:"-" md:"-" mapstructure:"-"`
CPE []CPEInfo `json:"cpe,omitempty" csv:"cpe" md:"cpe" mapstructure:"cpe"`
WordPress *WordPressInfo `json:"wordpress,omitempty" csv:"wordpress" md:"wordpress" mapstructure:"wordpress"`
}
type Trace struct {