From eec0fa69e547d9d4cecba9e94df9476f493f8a89 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 15 Sep 2022 23:09:29 +0530 Subject: [PATCH] Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change" This reverts commit 4634851eecf602004dc6360b5759a11c952b436d, reversing changes made to b7800f906c257db8ec3af5a620d8135349a56d7b. --- common/hashes/jarm/connpool.go | 47 +++++++++++++++++++++ common/hashes/jarm/jarmhash.go | 30 ++++++-------- common/hashes/jarm/onetimepool.go | 39 ----------------- common/httpx/response.go | 4 +- common/httpx/tls.go | 69 +++---------------------------- runner/runner.go | 32 ++++++++------ 6 files changed, 86 insertions(+), 135 deletions(-) create mode 100644 common/hashes/jarm/connpool.go diff --git a/common/hashes/jarm/connpool.go b/common/hashes/jarm/connpool.go new file mode 100644 index 0000000..56bc6b4 --- /dev/null +++ b/common/hashes/jarm/connpool.go @@ -0,0 +1,47 @@ +package jarm + +import ( + "net" + "sync" + + "go.uber.org/multierr" +) + +type inFlightConns struct { + sync.RWMutex + inflightConns map[net.Conn]struct{} +} + +func newInFlightConns() (*inFlightConns, error) { + return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil +} + +func (i *inFlightConns) Add(conn net.Conn) { + i.Lock() + defer i.Unlock() + + i.inflightConns[conn] = struct{}{} +} + +func (i *inFlightConns) Remove(conn net.Conn) { + i.Lock() + defer i.Unlock() + + delete(i.inflightConns, conn) +} + +func (i *inFlightConns) Close() error { + i.Lock() + defer i.Unlock() + + var errs []error + + for conn := range i.inflightConns { + if err := conn.Close(); err != nil { + errs = append(errs, err) + } + delete(i.inflightConns, conn) + } + + return multierr.Combine(errs...) +} diff --git a/common/hashes/jarm/jarmhash.go b/common/hashes/jarm/jarmhash.go index 5b45656..ad75640 100644 --- a/common/hashes/jarm/jarmhash.go +++ b/common/hashes/jarm/jarmhash.go @@ -13,10 +13,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" ) -const ( - poolCount = 3 - defaultPort = 443 -) +const defaultPort int = 443 type target struct { Host string @@ -24,15 +21,11 @@ type target struct { } // fingerprint probes a single host/port -func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string { +func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string { results := []string{} addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) - timeout := time.Duration(duration) * time.Second - - ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(duration*poolCount) * time.Second)) - defer cancel() - - pool, err := newOneTimePool(ctx, addr, poolCount) + // using connection pool as we need multiple probes + pool, err := newOneTimePool(context.Background(), addr, 3) if err != nil { return "" } @@ -42,18 +35,19 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string { go pool.Run() //nolint for _, probe := range jarm.GetProbes(t.Host, t.Port) { - conn, err := pool.Acquire(ctx) + conn, err := pool.Acquire(context.Background()) if err != nil { - return "" + continue } if conn == nil { - return "" + continue } _ = conn.SetWriteDeadline(time.Now().Add(timeout)) _, err = conn.Write(jarm.BuildProbe(probe)) if err != nil { + results = append(results, "") _ = conn.Close() - return "" + continue } _ = conn.SetReadDeadline(time.Now().Add(timeout)) buff := make([]byte, 1484) @@ -61,7 +55,8 @@ func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string { _ = conn.Close() ans, err := jarm.ParseServerHello(buff, probe) if err != nil { - return "" + results = append(results, "") + continue } results = append(results, ans) } @@ -81,5 +76,6 @@ func Jarm(dialer *fastdialer.Dialer, host string, duration int) string { if t.Port == 0 { t.Port = defaultPort } - return fingerprint(dialer, t, duration) + timeout := time.Duration(duration) * time.Second + return fingerprint(dialer, t, timeout) } diff --git a/common/hashes/jarm/onetimepool.go b/common/hashes/jarm/onetimepool.go index a30fe6e..c625ec8 100644 --- a/common/hashes/jarm/onetimepool.go +++ b/common/hashes/jarm/onetimepool.go @@ -3,10 +3,8 @@ package jarm import ( "context" "net" - "sync" "github.com/projectdiscovery/fastdialer/fastdialer" - "go.uber.org/multierr" ) // oneTimePool is a pool designed to create continous bare connections that are for one time only usage @@ -77,40 +75,3 @@ func (p *oneTimePool) Close() error { p.cancel() return p.InFlightConns.Close() } - -type inFlightConns struct { - sync.Mutex - inflightConns map[net.Conn]struct{} -} - -func newInFlightConns() (*inFlightConns, error) { - return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil -} - -func (i *inFlightConns) Add(conn net.Conn) { - i.Lock() - defer i.Unlock() - - i.inflightConns[conn] = struct{}{} -} - -func (i *inFlightConns) Remove(conn net.Conn) { - i.Lock() - defer i.Unlock() - - delete(i.inflightConns, conn) -} - -func (i *inFlightConns) Close() error { - i.Lock() - defer i.Unlock() - - var errs []error - for conn := range i.inflightConns { - if err := conn.Close(); err != nil { - errs = append(errs, err) - } - delete(i.inflightConns, conn) - } - return multierr.Combine(errs...) -} diff --git a/common/httpx/response.go b/common/httpx/response.go index fa561f7..99f6333 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -4,8 +4,8 @@ import ( "strings" "time" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/httputil" - "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" ) // Response contains the response to a server @@ -18,7 +18,7 @@ type Response struct { RawHeaders string Words int Lines int - TLSData *clients.Response + TLSData *cryptoutil.TLSData CSPData *CSPData HTTP2 bool Pipeline bool diff --git a/common/httpx/tls.go b/common/httpx/tls.go index cd7fd15..aa90c98 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -1,74 +1,15 @@ package httpx import ( - "crypto/tls" - "crypto/x509" - "net" "net/http" - "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" + "github.com/projectdiscovery/cryptoutil" ) -// versionToTLSVersionString converts tls version to version string -var versionToTLSVersionString = map[uint16]string{ - tls.VersionTLS10: "tls10", - tls.VersionTLS11: "tls11", - tls.VersionTLS12: "tls12", - tls.VersionTLS13: "tls13", -} - // TLSGrab fills the TLSData -func (h *HTTPX) TLSGrab(r *http.Response) *clients.Response { - if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { - return nil +func (h *HTTPX) TLSGrab(r *http.Response) *cryptoutil.TLSData { + if r.TLS != nil { + return cryptoutil.TLSGrab(r.TLS) } - host := r.Request.URL.Host - hostname, port, _ := net.SplitHostPort(host) - if hostname == "" { - hostname = host - } - if port == "" { - port = "443" - } - - tlsVersion := versionToTLSVersionString[r.TLS.Version] - tlsCipher := tls.CipherSuiteName(r.TLS.CipherSuite) - - leafCertificate := r.TLS.PeerCertificates[0] - response := &clients.Response{ - Host: hostname, - ProbeStatus: true, - Port: port, - Version: tlsVersion, - Cipher: tlsCipher, - TLSConnection: "ctls", - CertificateResponse: convertCertificateToResponse(hostname, leafCertificate), - ServerName: r.TLS.ServerName, - } - return response -} - -func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse { - response := &clients.CertificateResponse{ - SubjectAN: cert.DNSNames, - Emails: cert.EmailAddresses, - NotBefore: cert.NotBefore, - NotAfter: cert.NotAfter, - Expired: clients.IsExpired(cert.NotAfter), - SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId), - MisMatched: clients.IsMisMatchedCert(hostname, append(cert.DNSNames, cert.Subject.CommonName)), - WildCardCert: clients.IsWildCardCert(append(cert.DNSNames, cert.Subject.CommonName)), - IssuerCN: cert.Issuer.CommonName, - IssuerOrg: cert.Issuer.Organization, - SubjectCN: cert.Subject.CommonName, - SubjectOrg: cert.Subject.Organization, - FingerprintHash: clients.CertificateResponseFingerprintHash{ - MD5: clients.MD5Fingerprint(cert.Raw), - SHA1: clients.SHA1Fingerprint(cert.Raw), - SHA256: clients.SHA256Fingerprint(cert.Raw), - }, - } - response.IssuerDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawIssuer, cert.Issuer.String()) - response.SubjectDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawSubject, cert.Subject.String()) - return response + return nil } diff --git a/runner/runner.go b/runner/runner.go index 8167620..ad62735 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -27,7 +27,6 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" - "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -35,6 +34,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/clistats" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" @@ -796,14 +796,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false - for _, tt := range result.TLSData.SubjectAN { + for _, tt := range result.TLSData.DNSNames { if !r.testAndSet(tt) { continue } r.process(tt, wg, hp, protocol, scanopts, output) } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) + for _, tt := range result.TLSData.CommonName { + if !r.testAndSet(tt) { + continue + } + r.process(tt, wg, hp, protocol, scanopts, output) } } if scanopts.CSPProbe && result.CSPData != nil { @@ -835,14 +838,17 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false - for _, tt := range result.TLSData.SubjectAN { + for _, tt := range result.TLSData.DNSNames { if !r.testAndSet(tt) { continue } r.process(tt, wg, hp, protocol, scanopts, output) } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) + for _, tt := range result.TLSData.CommonName { + if !r.testAndSet(tt) { + continue + } + r.process(tt, wg, hp, protocol, scanopts, output) } } }(port, target, method, wantedProtocol) @@ -1567,12 +1573,12 @@ type Result struct { Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` ASN interface{} `json:"asn,omitempty" csv:"asn"` err error - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]string `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - Port string `json:"port,omitempty" csv:"port"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"` + Hashes map[string]string `json:"hash,omitempty" csv:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` + Port string `json:"port,omitempty" csv:"port"` raw string URL string `json:"url,omitempty" csv:"url"` Input string `json:"input,omitempty" csv:"input"`