Merge pull request #108 from projectdiscovery/feature-ips-cnames

adding cname + ips lookup in global cache

cmd/httpx/httpx.go

@@ -121,6 +121,7 @@ func main() {
 	scanopts.HTTP2Probe = options.HTTP2Probe
 	scanopts.OutputMethod = options.OutputMethod
 	scanopts.OutputIP = options.OutputIP
+	scanopts.OutputCName = options.OutputCName
 	scanopts.OutputCDN = options.OutputCDN
 	// output verb if more than one is specified
 	if len(scanopts.Methods) > 1 && !options.Silent {
@@ -321,6 +322,7 @@ type scanOptions struct {
 	Pipeline               bool
 	HTTP2Probe             bool
 	OutputIP               bool
+	OutputCName            bool
 	OutputCDN              bool
 }
 
@@ -492,7 +494,24 @@ retry:
 		builder.WriteString(fmt.Sprintf(" [%s]", ip))
 	}
 
-	isCDN := hp.CdnCheck(ip)
+	var (
+		ips    []string
+		cnames []string
+	)
+	dnsData, err := cache.GetDNSData(domain)
+	if dnsData != nil && err == nil {
+		ips = dnsData.IPs
+		cnames = dnsData.CNAMEs
+	} else {
+		ips = append(ips, ip)
+	}
+
+	if scanopts.OutputCName && len(cnames) > 0 {
+		// Print only the first CNAME (full list in json)
+		builder.WriteString(fmt.Sprintf(" [%s]", cnames[0]))
+  }
+	
+  isCDN := hp.CdnCheck(ip)
 	if scanopts.OutputCDN && isCDN {
 		builder.WriteString(" [cdn]")
 	}
@@ -529,9 +548,10 @@ retry:
 		HTTP2:         http2,
 		Method:        method,
 		IP:            ip,
+		IPs:           ips,
+		CNAMEs:        cnames,
 		CDN:           isCDN,
 	}
-
 }
 
 // Result of a scan
@@ -554,6 +574,8 @@ type Result struct {
 	HTTP2         bool           `json:"http2"`
 	Method        string         `json:"method"`
 	IP            string         `json:"ip"`
+	IPs           []string       `json:"ips"`
+	CNAMEs        []string       `json:"cnames,omitempty"`
 	CDN           bool           `json:"cdn"`
 }
 
@@ -609,6 +631,7 @@ type Options struct {
 	filterStatusCode          []int
 	OutputFilterContentLength string
 	OutputIP                  bool
+	OutputCName               bool
 	filterContentLength       []int
 	InputRawRequest           string
 	rawRequest                string
@@ -666,7 +689,9 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline")
 	flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
 	flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip")
+	flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname")
 	flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)")
+  
 	flag.Parse()
 
 	// Read the inputs and configure the logging

common/cache/cache.go

@@ -54,9 +54,9 @@ func New(options Options) (*Cache, error) {
 }
 
 // Lookup a hostname
-func (c *Cache) Lookup(hostname string) ([]string, error) {
+func (c *Cache) Lookup(hostname string) (*dns.Result, error) {
 	if ip := net.ParseIP(hostname); ip != nil {
-		return []string{hostname}, nil
+		return &dns.Result{IPs: []string{hostname}}, nil
 	}
 	hostnameBytes := []byte(hostname)
 	value, err := c.cache.Get(hostnameBytes)
@@ -64,17 +64,21 @@ func (c *Cache) Lookup(hostname string) ([]string, error) {
 		if len(err.Error()) != 15 {
 			return nil, err
 		}
-		results, err := c.dnsClient.Resolve(hostname)
+		result, err := c.dnsClient.Resolve(hostname)
 		if err != nil {
 			return nil, err
 		}
-		if results.TTL == 0 {
-			results.TTL = c.defaultExpirationTime
+		if result.TTL == 0 {
+			result.TTL = c.defaultExpirationTime
 		}
-		c.cache.Set(hostnameBytes, MarshalAddresses(results.IPs), results.TTL)
+		b, _ := result.Marshal()
+		c.cache.Set(hostnameBytes, b, result.TTL)
 
-		return results.IPs, nil
+		return &result, nil
 	}
 
-	return UnmarshalAddresses(value), nil
+	var result dns.Result
+	result.Unmarshal(value)
+
+	return &result, nil
 }

common/cache/dialer.go

@@ -7,9 +7,13 @@ import (
 	"time"
 
 	"github.com/coocood/freecache"
+	dns "github.com/projectdiscovery/httpx/common/resolver"
 )
 
-var dialerHistory *freecache.Cache
+var (
+	dialerHistory *freecache.Cache
+	cache         *Cache
+)
 
 // NoAddressFoundError occurs when no addresses are found for the host
 type NoAddressFoundError struct{}
@@ -23,7 +27,8 @@ type DialerFunc func(context.Context, string, string) (net.Conn, error)
 
 // NewDialer gets a new Dialer instance using a resolver cache
 func NewDialer(options Options) (DialerFunc, error) {
-	cache, err := New(options)
+	var err error
+	cache, err = New(options)
 	if err != nil {
 		return nil, err
 	}
@@ -38,17 +43,11 @@ func NewDialer(options Options) (DialerFunc, error) {
 
 		// we need to filter out empty records
 		hostname := address[:separator]
-		ips, err := cache.Lookup(hostname)
-		var finalIps []string
-		for _, ip := range ips {
-			if ip != "" {
-				finalIps = append(finalIps, ip)
-			}
-		}
-		if err != nil || len(finalIps) == 0 {
+		dnsResult, err := cache.Lookup(hostname)
+		if err != nil || len(dnsResult.IPs) == 0 {
 			return nil, &NoAddressFoundError{}
 		} // Dial to the IPs finally.
-		for _, ip := range ips {
+		for _, ip := range dnsResult.IPs {
 			conn, err = dialer.DialContext(ctx, network, ip+address[separator:])
 			if err == nil {
 				dialerHistory.Set([]byte(hostname), []byte(ip), 0)
@@ -67,3 +66,8 @@ func GetDialedIP(hostname string) string {
 	}
 	return string(v)
 }
+
+// GetDNSData cached by the resolver
+func GetDNSData(hostname string) (*dns.Result, error) {
+	return cache.Lookup(hostname)
+}

common/cache/utils.go

@@ -1,13 +0,0 @@
-package cache
-
-import "strings"
-
-var Separator = "-"
-
-func MarshalAddresses(ips []string) []byte {
-	return []byte(strings.Join(ips, Separator))
-}
-
-func UnmarshalAddresses(data []byte) []string {
-	return strings.Split(string(data), Separator)
-}

common/resolver/client.go

@@ -1,8 +1,11 @@
 package dns
 
 import (
+	"bytes"
+	"encoding/gob"
 	"errors"
 	"math/rand"
+	"strings"
 	"time"
 
 	"github.com/miekg/dns"
@@ -18,8 +21,31 @@ type Client struct {
 
 // Result containing ip and time to live
 type Result struct {
-	IPs []string
-	TTL int
+	IPs    []string
+	CNAMEs []string
+	TTL    int
+}
+
+// Marshal structure to bytes
+func (r *Result) Marshal() ([]byte, error) {
+	var b bytes.Buffer
+	enc := gob.NewEncoder(&b)
+	err := enc.Encode(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// Unmarshal structure
+func (r *Result) Unmarshal(b []byte) error {
+	dec := gob.NewDecoder(bytes.NewBuffer(b))
+	err := dec.Decode(&r)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // New creates a new dns client
@@ -57,9 +83,17 @@ func (c *Client) Resolve(host string) (Result, error) {
 			return result, errors.New(dns.RcodeToString[answer.Rcode])
 		}
 		for _, record := range answer.Answer {
-			if t, ok := record.(*dns.A); ok {
-				result.IPs = append(result.IPs, t.A.String())
-				result.TTL = int(t.Header().Ttl)
+			switch t := record.(type) {
+			case *dns.A:
+				ip := t.A.String()
+				if ip != "" {
+					result.IPs = append(result.IPs, t.A.String())
+					result.TTL = int(t.Header().Ttl)
+				}
+			case *dns.CNAME:
+				if t.Target != "" {
+					result.CNAMEs = append(result.CNAMEs, strings.TrimSuffix(t.Target, "."))
+				}
 			}
 		}
 		return result, nil