diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 12a4760..58d3ee4 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -2,7 +2,9 @@ package stringz import ( "bytes" + "crypto/md5" "encoding/base64" + "encoding/hex" "errors" "net/http" "net/url" @@ -121,12 +123,19 @@ func murmurhash(data []byte) int32 { return int32(hasher.Sum32()) } -func FaviconHash(data []byte) (int32, error) { +// md5Hash returns the md5 hash of the data +func md5Hash(data []byte) string { + hasher := md5.New() + hasher.Write(data) + return hex.EncodeToString(hasher.Sum(nil)) +} + +func FaviconHash(data []byte) (int32, string, error) { if isContentTypeImage(data) { - return murmurhash(data), nil + return murmurhash(data), md5Hash(data), nil } - return 0, errors.New("content type is not image") + return 0, "", errors.New("content type is not image") } func InsertInto(s string, interval int, sep rune) string { diff --git a/runner/runner.go b/runner/runner.go index 17932d1..a66663f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1948,11 +1948,11 @@ retry: builder.WriteRune(']') } - var faviconMMH3, faviconPath, faviconURL string + var faviconMMH3, faviconMD5, faviconPath, faviconURL string var faviconData []byte if scanopts.Favicon { var err error - faviconMMH3, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true) + faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true) if err == nil { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -2199,6 +2199,7 @@ retry: Technologies: technologies, FinalURL: finalURL, FavIconMMH3: faviconMMH3, + FavIconMD5: faviconMD5, FaviconPath: faviconPath, FaviconURL: faviconURL, Hashes: hashesMap, @@ -2257,11 +2258,11 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) { return pHash.GetHash(), nil } -func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, []byte, string, error) { +func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, string, []byte, string, error) { // Check if current URI is ending with .ico => use current body without additional requests if path.Ext(req.URL.Path) == ".ico" { - hash, err := r.calculateFaviconHashWithRaw(currentResp) - return hash, req.URL.Path, currentResp, "", err + MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(currentResp) + return MMH3Hash, MD5Hash, req.URL.Path, currentResp, "", err } // search in the response of the requested path for element and rel shortcut/mask/apple-touch icon @@ -2269,12 +2270,12 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, // if not, any of link from other icons can be requested potentialURLs, err := extractPotentialFavIconsURLs(currentResp) if err != nil { - return "", "", nil, "", err + return "", "", "", nil, "", err } clone := req.Clone(context.Background()) - var faviconHash, faviconPath, faviconURL string + var faviconMMH3, faviconMD5, faviconPath, faviconURL string var faviconData []byte errCount := 0 if len(potentialURLs) == 0 && defaultProbe { @@ -2309,25 +2310,26 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, errCount++ continue } - hash, err := r.calculateFaviconHashWithRaw(resp.Data) + MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(resp.Data) if err != nil { continue } faviconURL = clone.URL.String() faviconPath = potentialURL - faviconHash = hash + faviconMMH3 = MMH3Hash + faviconMD5 = MD5Hash faviconData = resp.Data break } - return faviconHash, faviconPath, faviconData, faviconURL, nil + return faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, nil } -func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) { - hashNum, err := stringz.FaviconHash(data) +func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, string, error) { + hashNum, md5Hash, err := stringz.FaviconHash(data) if err != nil { - return "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) + return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err) } - return fmt.Sprintf("%d", hashNum), nil + return fmt.Sprintf("%d", hashNum), md5Hash, nil } func extractPotentialFavIconsURLs(resp []byte) ([]string, error) { diff --git a/runner/types.go b/runner/types.go index 87d5f79..9c986ae 100644 --- a/runner/types.go +++ b/runner/types.go @@ -59,6 +59,7 @@ type Result struct { Host string `json:"host,omitempty" csv:"host"` Path string `json:"path,omitempty" csv:"path"` FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon"` FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path"` FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url"` FinalURL string `json:"final_url,omitempty" csv:"final_url"`