admin/projectdiscovery-httpx
1
0
Fork 0
mirror of https://github.com/projectdiscovery/httpx synced 2026-06-08 16:50:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change"

Browse Source 
This reverts commit 4634851eec, reversing
changes made to b7800f906c.
This commit is contained in:
sandeep
2022-09-15 23:09:29 +05:30
parent 4634851eec
commit eec0fa69e5
6 changed files with 86 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/hashes/jarm/connpool.go
+47
View File
@@ -0,0 +1,47 @@
package jarm
import (
"net"
"sync"
"go.uber.org/multierr"
)
type inFlightConns struct {
sync.RWMutex
inflightConns map[net.Conn]struct{}
}
func newInFlightConns() (*inFlightConns, error) {
return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
}
func (i *inFlightConns) Add(conn net.Conn) {
i.Lock()
defer i.Unlock()
i.inflightConns[conn] = struct{}{}
}
func (i *inFlightConns) Remove(conn net.Conn) {
i.Lock()
defer i.Unlock()
delete(i.inflightConns, conn)
}
func (i *inFlightConns) Close() error {
i.Lock()
defer i.Unlock()
var errs []error
for conn := range i.inflightConns {
if err := conn.Close(); err != nil {
errs = append(errs, err)
}
delete(i.inflightConns, conn)
}
return multierr.Combine(errs...)
}
common/hashes/jarm/jarmhash.go
+13 -17
View File
@@ -13,10 +13,7 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
)
const (
poolCount = 3
defaultPort = 443
)
const defaultPort int = 443
type target struct {
Host string
@@ -24,15 +21,11 @@ type target struct {
}
// fingerprint probes a single host/port
func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string {
results := []string{}
addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port))
timeout := time.Duration(duration) * time.Second
ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(duration*poolCount) * time.Second))
defer cancel()
pool, err := newOneTimePool(ctx, addr, poolCount)
// using connection pool as we need multiple probes
pool, err := newOneTimePool(context.Background(), addr, 3)
if err != nil {
return ""
}
@@ -42,18 +35,19 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
go pool.Run() //nolint
for _, probe := range jarm.GetProbes(t.Host, t.Port) {
conn, err := pool.Acquire(ctx)
conn, err := pool.Acquire(context.Background())
if err != nil {
return ""
continue
}
if conn == nil {
return ""
continue
}
_ = conn.SetWriteDeadline(time.Now().Add(timeout))
_, err = conn.Write(jarm.BuildProbe(probe))
if err != nil {
results = append(results, "")
_ = conn.Close()
return ""
continue
}
_ = conn.SetReadDeadline(time.Now().Add(timeout))
buff := make([]byte, 1484)
@@ -61,7 +55,8 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string {
_ = conn.Close()
ans, err := jarm.ParseServerHello(buff, probe)
if err != nil {
return ""
results = append(results, "")
continue
}
results = append(results, ans)
}
@@ -81,5 +76,6 @@ func Jarm(dialer *fastdialer.Dialer, host string, duration int) string {
if t.Port == 0 {
t.Port = defaultPort
}
return fingerprint(dialer, t, duration)
timeout := time.Duration(duration) * time.Second
return fingerprint(dialer, t, timeout)
}
common/hashes/jarm/onetimepool.go
-39
View File
@@ -3,10 +3,8 @@ package jarm
import (
"context"
"net"
"sync"
"github.com/projectdiscovery/fastdialer/fastdialer"
"go.uber.org/multierr"
)
// oneTimePool is a pool designed to create continous bare connections that are for one time only usage
@@ -77,40 +75,3 @@ func (p *oneTimePool) Close() error {
p.cancel()
return p.InFlightConns.Close()
}
type inFlightConns struct {
sync.Mutex
inflightConns map[net.Conn]struct{}
}
func newInFlightConns() (*inFlightConns, error) {
return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil
}
func (i *inFlightConns) Add(conn net.Conn) {
i.Lock()
defer i.Unlock()
i.inflightConns[conn] = struct{}{}
}
func (i *inFlightConns) Remove(conn net.Conn) {
i.Lock()
defer i.Unlock()
delete(i.inflightConns, conn)
}
func (i *inFlightConns) Close() error {
i.Lock()
defer i.Unlock()
var errs []error
for conn := range i.inflightConns {
if err := conn.Close(); err != nil {
errs = append(errs, err)
}
delete(i.inflightConns, conn)
}
return multierr.Combine(errs...)
}
common/httpx/response.go
+2 -2
View File
@@ -4,8 +4,8 @@ import (
"strings"
"time"
"github.com/projectdiscovery/cryptoutil"
"github.com/projectdiscovery/httputil"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
)
// Response contains the response to a server
@@ -18,7 +18,7 @@ type Response struct {
RawHeaders string
Words int
Lines int
TLSData *clients.Response
TLSData *cryptoutil.TLSData
CSPData *CSPData
HTTP2 bool
Pipeline bool
common/httpx/tls.go
+5 -64
View File
@@ -1,74 +1,15 @@
package httpx
import (
"crypto/tls"
"crypto/x509"
"net"
"net/http"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
"github.com/projectdiscovery/cryptoutil"
)
// versionToTLSVersionString converts tls version to version string
var versionToTLSVersionString = map[uint16]string{
tls.VersionTLS10: "tls10",
tls.VersionTLS11: "tls11",
tls.VersionTLS12: "tls12",
tls.VersionTLS13: "tls13",
}
// TLSGrab fills the TLSData
func (h *HTTPX) TLSGrab(r *http.Response) *clients.Response {
if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 {
return nil
func (h *HTTPX) TLSGrab(r *http.Response) *cryptoutil.TLSData {
if r.TLS != nil {
return cryptoutil.TLSGrab(r.TLS)
}
host := r.Request.URL.Host
hostname, port, _ := net.SplitHostPort(host)
if hostname == "" {
hostname = host
}
if port == "" {
port = "443"
}
tlsVersion := versionToTLSVersionString[r.TLS.Version]
tlsCipher := tls.CipherSuiteName(r.TLS.CipherSuite)
leafCertificate := r.TLS.PeerCertificates[0]
response := &clients.Response{
Host: hostname,
ProbeStatus: true,
Port: port,
Version: tlsVersion,
Cipher: tlsCipher,
TLSConnection: "ctls",
CertificateResponse: convertCertificateToResponse(hostname, leafCertificate),
ServerName: r.TLS.ServerName,
}
return response
}
func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse {
response := &clients.CertificateResponse{
SubjectAN: cert.DNSNames,
Emails: cert.EmailAddresses,
NotBefore: cert.NotBefore,
NotAfter: cert.NotAfter,
Expired: clients.IsExpired(cert.NotAfter),
SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId),
MisMatched: clients.IsMisMatchedCert(hostname, append(cert.DNSNames, cert.Subject.CommonName)),
WildCardCert: clients.IsWildCardCert(append(cert.DNSNames, cert.Subject.CommonName)),
IssuerCN: cert.Issuer.CommonName,
IssuerOrg: cert.Issuer.Organization,
SubjectCN: cert.Subject.CommonName,
SubjectOrg: cert.Subject.Organization,
FingerprintHash: clients.CertificateResponseFingerprintHash{
MD5: clients.MD5Fingerprint(cert.Raw),
SHA1: clients.SHA1Fingerprint(cert.Raw),
SHA256: clients.SHA256Fingerprint(cert.Raw),
},
}
response.IssuerDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawIssuer, cert.Issuer.String())
response.SubjectDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawSubject, cert.Subject.String())
return response
return nil
}
runner/runner.go
+19 -13
View File
@@ -27,7 +27,6 @@ import (
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/httpx/common/customextract"
"github.com/projectdiscovery/httpx/common/hashes/jarm"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
"github.com/ammario/ipisp/v2"
"github.com/bluele/gcache"
@@ -35,6 +34,7 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/clistats"
"github.com/projectdiscovery/cryptoutil"
"github.com/projectdiscovery/goconfig"
"github.com/projectdiscovery/httpx/common/hashes"
"github.com/projectdiscovery/retryablehttp-go"
@@ -796,14 +796,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
output <- result
if scanopts.TLSProbe && result.TLSData != nil {
scanopts.TLSProbe = false
for _, tt := range result.TLSData.SubjectAN {
for _, tt := range result.TLSData.DNSNames {
if !r.testAndSet(tt) {
continue
}
r.process(tt, wg, hp, protocol, scanopts, output)
}
if r.testAndSet(result.TLSData.SubjectCN) {
r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
for _, tt := range result.TLSData.CommonName {
if !r.testAndSet(tt) {
continue
}
r.process(tt, wg, hp, protocol, scanopts, output)
}
}
if scanopts.CSPProbe && result.CSPData != nil {
@@ -835,14 +838,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
output <- result
if scanopts.TLSProbe && result.TLSData != nil {
scanopts.TLSProbe = false
for _, tt := range result.TLSData.SubjectAN {
for _, tt := range result.TLSData.DNSNames {
if !r.testAndSet(tt) {
continue
}
r.process(tt, wg, hp, protocol, scanopts, output)
}
if r.testAndSet(result.TLSData.SubjectCN) {
r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output)
for _, tt := range result.TLSData.CommonName {
if !r.testAndSet(tt) {
continue
}
r.process(tt, wg, hp, protocol, scanopts, output)
}
}
}(port, target, method, wantedProtocol)
@@ -1567,12 +1573,12 @@ type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN interface{} `json:"asn,omitempty" csv:"asn"`
err error
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
Hashes map[string]string `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
Port string `json:"port,omitempty" csv:"port"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
TLSData *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"`
Hashes map[string]string `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
Port string `json:"port,omitempty" csv:"port"`
raw string
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`