diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 418787a..6b245f5 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -121,6 +121,7 @@ func main() { scanopts.HTTP2Probe = options.HTTP2Probe scanopts.OutputMethod = options.OutputMethod scanopts.OutputIP = options.OutputIP + scanopts.OutputCName = options.OutputCName scanopts.OutputCDN = options.OutputCDN // output verb if more than one is specified if len(scanopts.Methods) > 1 && !options.Silent { @@ -321,6 +322,7 @@ type scanOptions struct { Pipeline bool HTTP2Probe bool OutputIP bool + OutputCName bool OutputCDN bool } @@ -492,7 +494,24 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", ip)) } - isCDN := hp.CdnCheck(ip) + var ( + ips []string + cnames []string + ) + dnsData, err := cache.GetDNSData(domain) + if dnsData != nil && err == nil { + ips = dnsData.IPs + cnames = dnsData.CNAMEs + } else { + ips = append(ips, ip) + } + + if scanopts.OutputCName && len(cnames) > 0 { + // Print only the first CNAME (full list in json) + builder.WriteString(fmt.Sprintf(" [%s]", cnames[0])) + } + + isCDN := hp.CdnCheck(ip) if scanopts.OutputCDN && isCDN { builder.WriteString(" [cdn]") } @@ -529,9 +548,10 @@ retry: HTTP2: http2, Method: method, IP: ip, + IPs: ips, + CNAMEs: cnames, CDN: isCDN, } - } // Result of a scan @@ -554,6 +574,8 @@ type Result struct { HTTP2 bool `json:"http2"` Method string `json:"method"` IP string `json:"ip"` + IPs []string `json:"ips"` + CNAMEs []string `json:"cnames,omitempty"` CDN bool `json:"cdn"` } @@ -609,6 +631,7 @@ type Options struct { filterStatusCode []int OutputFilterContentLength string OutputIP bool + OutputCName bool filterContentLength []int InputRawRequest string rawRequest string @@ -666,7 +689,9 @@ func ParseOptions() *Options { flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline") flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip") + flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname") flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)") + flag.Parse() // Read the inputs and configure the logging diff --git a/common/cache/cache.go b/common/cache/cache.go index db37126..ddc8300 100644 --- a/common/cache/cache.go +++ b/common/cache/cache.go @@ -54,9 +54,9 @@ func New(options Options) (*Cache, error) { } // Lookup a hostname -func (c *Cache) Lookup(hostname string) ([]string, error) { +func (c *Cache) Lookup(hostname string) (*dns.Result, error) { if ip := net.ParseIP(hostname); ip != nil { - return []string{hostname}, nil + return &dns.Result{IPs: []string{hostname}}, nil } hostnameBytes := []byte(hostname) value, err := c.cache.Get(hostnameBytes) @@ -64,17 +64,21 @@ func (c *Cache) Lookup(hostname string) ([]string, error) { if len(err.Error()) != 15 { return nil, err } - results, err := c.dnsClient.Resolve(hostname) + result, err := c.dnsClient.Resolve(hostname) if err != nil { return nil, err } - if results.TTL == 0 { - results.TTL = c.defaultExpirationTime + if result.TTL == 0 { + result.TTL = c.defaultExpirationTime } - c.cache.Set(hostnameBytes, MarshalAddresses(results.IPs), results.TTL) + b, _ := result.Marshal() + c.cache.Set(hostnameBytes, b, result.TTL) - return results.IPs, nil + return &result, nil } - return UnmarshalAddresses(value), nil + var result dns.Result + result.Unmarshal(value) + + return &result, nil } diff --git a/common/cache/dialer.go b/common/cache/dialer.go index 952fd38..8c4de21 100644 --- a/common/cache/dialer.go +++ b/common/cache/dialer.go @@ -7,9 +7,13 @@ import ( "time" "github.com/coocood/freecache" + dns "github.com/projectdiscovery/httpx/common/resolver" ) -var dialerHistory *freecache.Cache +var ( + dialerHistory *freecache.Cache + cache *Cache +) // NoAddressFoundError occurs when no addresses are found for the host type NoAddressFoundError struct{} @@ -23,7 +27,8 @@ type DialerFunc func(context.Context, string, string) (net.Conn, error) // NewDialer gets a new Dialer instance using a resolver cache func NewDialer(options Options) (DialerFunc, error) { - cache, err := New(options) + var err error + cache, err = New(options) if err != nil { return nil, err } @@ -38,17 +43,11 @@ func NewDialer(options Options) (DialerFunc, error) { // we need to filter out empty records hostname := address[:separator] - ips, err := cache.Lookup(hostname) - var finalIps []string - for _, ip := range ips { - if ip != "" { - finalIps = append(finalIps, ip) - } - } - if err != nil || len(finalIps) == 0 { + dnsResult, err := cache.Lookup(hostname) + if err != nil || len(dnsResult.IPs) == 0 { return nil, &NoAddressFoundError{} } // Dial to the IPs finally. - for _, ip := range ips { + for _, ip := range dnsResult.IPs { conn, err = dialer.DialContext(ctx, network, ip+address[separator:]) if err == nil { dialerHistory.Set([]byte(hostname), []byte(ip), 0) @@ -67,3 +66,8 @@ func GetDialedIP(hostname string) string { } return string(v) } + +// GetDNSData cached by the resolver +func GetDNSData(hostname string) (*dns.Result, error) { + return cache.Lookup(hostname) +} diff --git a/common/cache/utils.go b/common/cache/utils.go deleted file mode 100644 index a5facab..0000000 --- a/common/cache/utils.go +++ /dev/null @@ -1,13 +0,0 @@ -package cache - -import "strings" - -var Separator = "-" - -func MarshalAddresses(ips []string) []byte { - return []byte(strings.Join(ips, Separator)) -} - -func UnmarshalAddresses(data []byte) []string { - return strings.Split(string(data), Separator) -} diff --git a/common/resolver/client.go b/common/resolver/client.go index ffdd252..367890c 100644 --- a/common/resolver/client.go +++ b/common/resolver/client.go @@ -1,8 +1,11 @@ package dns import ( + "bytes" + "encoding/gob" "errors" "math/rand" + "strings" "time" "github.com/miekg/dns" @@ -18,8 +21,31 @@ type Client struct { // Result containing ip and time to live type Result struct { - IPs []string - TTL int + IPs []string + CNAMEs []string + TTL int +} + +// Marshal structure to bytes +func (r *Result) Marshal() ([]byte, error) { + var b bytes.Buffer + enc := gob.NewEncoder(&b) + err := enc.Encode(r) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +// Unmarshal structure +func (r *Result) Unmarshal(b []byte) error { + dec := gob.NewDecoder(bytes.NewBuffer(b)) + err := dec.Decode(&r) + if err != nil { + return err + } + return nil } // New creates a new dns client @@ -57,9 +83,17 @@ func (c *Client) Resolve(host string) (Result, error) { return result, errors.New(dns.RcodeToString[answer.Rcode]) } for _, record := range answer.Answer { - if t, ok := record.(*dns.A); ok { - result.IPs = append(result.IPs, t.A.String()) - result.TTL = int(t.Header().Ttl) + switch t := record.(type) { + case *dns.A: + ip := t.A.String() + if ip != "" { + result.IPs = append(result.IPs, t.A.String()) + result.TTL = int(t.Header().Ttl) + } + case *dns.CNAME: + if t.Target != "" { + result.CNAMEs = append(result.CNAMEs, strings.TrimSuffix(t.Target, ".")) + } } } return result, nil