diff --git a/README.md b/README.md index 5847f1a..6fed2e7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi | Favicon Hash | false | Probe Status | false | | Body Hash | true | Header Hash | true | | Redirect chain | false | URL Scheme | true | +| JARM Hash | false | | | # Installation Instructions @@ -93,6 +94,7 @@ PROBES: -location display response redirect location -favicon display mmh3 hash for '/favicon.ico' file -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) + -jarm display jarm fingerprint hash -rt, -response-time display response time -lc, -line-count display response body line count -wc, -word-count display response body word count @@ -334,6 +336,33 @@ https://mta-sts.hackerone.com/favicon.ico [-1700323260] https://www.hackerone.com/favicon.ico [778073381] ``` +### [JARM Fingerprint](https://github.com/salesforce/jarm) + + +```console +subfinder -d hackerone.com -silent | httpx -jarm + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.2.1 + + projectdiscovery.io + +Use with caution. You are responsible for your actions. +Developers assume no liability and are not responsible for any misuse or damage. +https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] +https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7] +https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] +https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5] +``` + + ### Path Probe diff --git a/common/hashes/hashes.go b/common/hashes/hashes.go index 884968d..5a2a160 100644 --- a/common/hashes/hashes.go +++ b/common/hashes/hashes.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/mfonda/simhash" "github.com/spaolacci/murmur3" ) diff --git a/common/hashes/jarmhash.go b/common/hashes/jarmhash.go new file mode 100644 index 0000000..e1d48d1 --- /dev/null +++ b/common/hashes/jarmhash.go @@ -0,0 +1,93 @@ +package hashes + +import ( + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/RumbleDiscovery/jarm-go" + "github.com/projectdiscovery/httpx/common/regexhelper" + "golang.org/x/net/proxy" +) + +const defaultPort int = 443 + +var DefualtBackoff = func(r, m int) time.Duration { + return time.Second +} + +type target struct { + Host string + Port int + Retries int + Backoff func(r, m int) time.Duration +} + +// fingerprint probes a single host/port +func fingerprint(t target, duration int) string { + timeout := time.Duration(duration) * time.Second + results := []string{} + for _, probe := range jarm.GetProbes(t.Host, t.Port) { + dialer := proxy.FromEnvironmentUsing(&net.Dialer{Timeout: timeout}) + addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) + c := net.Conn(nil) + n := 0 + for c == nil && n <= t.Retries { + // Ignoring error since error message was already being dropped. + // Also, if theres an error, c == nil. + if c, _ = dialer.Dial("tcp", addr); c != nil || t.Retries == 0 { + break + } + bo := t.Backoff + if bo == nil { + bo = DefualtBackoff + } + time.Sleep(bo(n, t.Retries)) + n++ + } + if c == nil { + return "" + } + data := jarm.BuildProbe(probe) + _ = c.SetWriteDeadline(time.Now().Add(timeout)) + _, err := c.Write(data) + if err != nil { + results = append(results, "") + c.Close() + continue + } + _ = c.SetReadDeadline(time.Now().Add(timeout)) + buff := make([]byte, 1484) + _, _ = c.Read(buff) + c.Close() + ans, err := jarm.ParseServerHello(buff, probe) + if err != nil { + results = append(results, "") + continue + } + results = append(results, ans) + } + return jarm.RawHashToFuzzyHash(strings.Join(results, ",")) +} +func Jarm(host string, duration int) string { + t := target{} + if u, err := url.Parse(host); err == nil { + if u.Scheme == "http" { + return "" + } + t.Host = u.Hostname() + port, _ := strconv.Atoi(u.Port()) + t.Port = port + } + if t.Port == 0 { + t.Port = defaultPort + } + hash := fingerprint(t, duration) + if regexhelper.JarmHashRegex.MatchString(hash) { + return "" + } + return hash +} diff --git a/common/regexhelper/regex.go b/common/regexhelper/regex.go new file mode 100644 index 0000000..7bdd81a --- /dev/null +++ b/common/regexhelper/regex.go @@ -0,0 +1,7 @@ +package regexhelper + +import "regexp" + +var ( + JarmHashRegex = regexp.MustCompile("(?m)0{62}") +) diff --git a/go.mod b/go.mod index 97619cd..f98b287 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( ) require ( + github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect github.com/ammario/ipisp/v2 v2.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect diff --git a/go.sum b/go.sum index dfbac3f..986f755 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,9 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= +github.com/RumbleDiscovery/jarm-go v0.0.6/go.mod h1:dXV7z5vBXQI0cNaHXwzGtq2PJ2LgM3XgcFiX32FU3bg= +github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= @@ -32,6 +35,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -85,6 +89,7 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= @@ -185,6 +190,7 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -228,6 +234,7 @@ go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6m golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= @@ -243,6 +250,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -256,6 +264,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -270,6 +279,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/runner/options.go b/runner/options.go index 08c14ef..602faf0 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,12 +1,11 @@ package runner import ( - "github.com/projectdiscovery/httpx/common/slice" "math" "os" "regexp" "strings" - + "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" @@ -228,6 +227,7 @@ type Options struct { OutputFilterWordsCount string filterWordsCount []int Hashes string + Jarm bool Asn bool } @@ -250,6 +250,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"), flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"), flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"), + flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"), flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"), flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"), flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"), diff --git a/runner/runner.go b/runner/runner.go index c96ba22..73c0270 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1291,7 +1291,17 @@ retry: } builder.WriteRune(']') } - + jarmhash := "" + if r.options.Jarm { + jarmhash = hashes.Jarm(fullURL,r.options.Timeout) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(jarmhash).String()) + } else { + builder.WriteString(fmt.Sprint(jarmhash)) + } + builder.WriteRune(']') + } if scanopts.OutputWordsCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1397,6 +1407,7 @@ retry: FinalURL: finalURL, FavIconMMH3: faviconMMH3, Hashes: hashesMap, + Jarm: jarmhash, Lines: resp.Lines, Words: resp.Words, ASN: asnResponse, @@ -1466,6 +1477,7 @@ type Result struct { ASN interface{} `json:"asn,omitempty" csv:"asn"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` } // JSON the result