mirror of
https://github.com/projectdiscovery/httpx
synced 2026-06-08 16:50:17 +00:00
Dogan Can Bakir
@@ -17,8 +17,9 @@ updates:
|
||||
include: "scope"
|
||||
labels:
|
||||
- "Type: Maintenance"
|
||||
allow:
|
||||
- dependency-name: "github.com/projectdiscovery/*"
|
||||
groups:
|
||||
modules:
|
||||
patterns: ["github.com/projectdiscovery/*"]
|
||||
|
||||
# # Maintain dependencies for GitHub Actions
|
||||
# - package-ecosystem: "github-actions"
|
||||
@@ -42,4 +43,4 @@ updates:
|
||||
# prefix: "chore"
|
||||
# include: "scope"
|
||||
# labels:
|
||||
# - "Type: Maintenance"
|
||||
# - "Type: Maintenance"
|
||||
|
||||
@@ -164,7 +164,13 @@ func New(options *Options) (*HTTPX, error) {
|
||||
}
|
||||
|
||||
if httpx.Options.HTTPProxy != "" {
|
||||
proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy)
|
||||
httpx.Options.Proxy = httpx.Options.HTTPProxy
|
||||
} else if httpx.Options.SocksProxy != "" {
|
||||
httpx.Options.Proxy = httpx.Options.SocksProxy
|
||||
}
|
||||
|
||||
if httpx.Options.Proxy != "" {
|
||||
proxyURL, parseErr := url.Parse(httpx.Options.Proxy)
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
|
||||
@@ -13,12 +13,15 @@ import (
|
||||
type Options struct {
|
||||
RandomAgent bool
|
||||
DefaultUserAgent string
|
||||
HTTPProxy string
|
||||
SocksProxy string
|
||||
Threads int
|
||||
CdnCheck string
|
||||
ExcludeCdn bool
|
||||
ExtractFqdn bool
|
||||
Proxy string
|
||||
// Deprecated: use Proxy
|
||||
HTTPProxy string
|
||||
// Deprecated: use Proxy
|
||||
SocksProxy string
|
||||
Threads int
|
||||
CdnCheck string
|
||||
ExcludeCdn bool
|
||||
ExtractFqdn bool
|
||||
// Timeout is the maximum time to wait for the request
|
||||
Timeout time.Duration
|
||||
// RetryMax is the maximum number of retries
|
||||
|
||||
@@ -22,31 +22,31 @@ require (
|
||||
github.com/projectdiscovery/asnmap v1.1.1
|
||||
github.com/projectdiscovery/cdncheck v1.1.0
|
||||
github.com/projectdiscovery/clistats v0.1.1
|
||||
github.com/projectdiscovery/dsl v0.2.5
|
||||
github.com/projectdiscovery/dsl v0.3.13
|
||||
github.com/projectdiscovery/fastdialer v0.3.0
|
||||
github.com/projectdiscovery/fdmax v0.0.4
|
||||
github.com/projectdiscovery/goconfig v0.0.1
|
||||
github.com/projectdiscovery/goflags v0.1.64
|
||||
github.com/projectdiscovery/gologger v1.1.33
|
||||
github.com/projectdiscovery/hmap v0.0.70
|
||||
github.com/projectdiscovery/goflags v0.1.67
|
||||
github.com/projectdiscovery/gologger v1.1.41
|
||||
github.com/projectdiscovery/hmap v0.0.76
|
||||
github.com/projectdiscovery/mapcidr v1.1.34
|
||||
github.com/projectdiscovery/networkpolicy v0.1.1
|
||||
github.com/projectdiscovery/ratelimit v0.0.54
|
||||
github.com/projectdiscovery/rawhttp v0.1.68
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.88
|
||||
github.com/projectdiscovery/tlsx v1.1.7
|
||||
github.com/projectdiscovery/useragent v0.0.69
|
||||
github.com/projectdiscovery/utils v0.3.0
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.22
|
||||
github.com/projectdiscovery/ratelimit v0.0.68
|
||||
github.com/projectdiscovery/rawhttp v0.1.84
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.96
|
||||
github.com/projectdiscovery/tlsx v1.1.8
|
||||
github.com/projectdiscovery/useragent v0.0.87
|
||||
github.com/projectdiscovery/utils v0.4.8
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.12
|
||||
github.com/rs/xid v1.5.0
|
||||
github.com/spaolacci/murmur3 v1.1.0
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db
|
||||
go.etcd.io/bbolt v1.3.10 // indirect
|
||||
go.uber.org/multierr v1.11.0
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
|
||||
golang.org/x/net v0.33.0
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/net v0.34.0
|
||||
golang.org/x/sys v0.29.0 // indirect
|
||||
golang.org/x/text v0.21.0
|
||||
)
|
||||
|
||||
@@ -119,7 +119,7 @@ require (
|
||||
github.com/projectdiscovery/freeport v0.0.7 // indirect
|
||||
github.com/projectdiscovery/gostruct v0.0.2 // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||
github.com/projectdiscovery/retryabledns v1.0.87 // indirect
|
||||
github.com/projectdiscovery/retryabledns v1.0.94 // indirect
|
||||
github.com/refraction-networking/utls v1.6.7 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
@@ -151,11 +151,11 @@ require (
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zcalusic/sysinfo v1.0.2 // indirect
|
||||
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/crypto v0.32.0 // indirect
|
||||
golang.org/x/mod v0.17.0 // indirect
|
||||
golang.org/x/oauth2 v0.18.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
golang.org/x/term v0.27.0 // indirect
|
||||
golang.org/x/term v0.28.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
|
||||
@@ -250,8 +250,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/
|
||||
github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc=
|
||||
github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE=
|
||||
github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
|
||||
github.com/projectdiscovery/dsl v0.2.5 h1:SJzIPGryApvNuAMa/VCe2yPzIDbkoz/pyH8Zt3Wlk+0=
|
||||
github.com/projectdiscovery/dsl v0.2.5/go.mod h1:aLyfr+br+cgwhHUWdQkIYJz+riWABaJ76jKcPYkcWYE=
|
||||
github.com/projectdiscovery/dsl v0.3.13 h1:3/2Cd7bl5qS3bpVIKK4Qns2f8/XKPlj82SsztHplHg0=
|
||||
github.com/projectdiscovery/dsl v0.3.13/go.mod h1:nWkkkfreA+P0nc0O9lalqu5lieIirSYNAD4W9daKd+U=
|
||||
github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk=
|
||||
github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ=
|
||||
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
|
||||
@@ -260,38 +260,38 @@ github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxys
|
||||
github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k=
|
||||
github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s=
|
||||
github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw=
|
||||
github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI=
|
||||
github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4=
|
||||
github.com/projectdiscovery/gologger v1.1.33 h1:wQxaQ8p/0Rx89lowBp0PnY2QSWiqf9QW1vGYAllsVJ4=
|
||||
github.com/projectdiscovery/gologger v1.1.33/go.mod h1:P/WwqKstshQATJxN39V0KJ9ZuiGLOizmSqHIYrrz1T4=
|
||||
github.com/projectdiscovery/goflags v0.1.67 h1:WOS1gdirH1Xrig8tYM9QC5UD/dfgABUcvrL0d8/fCPA=
|
||||
github.com/projectdiscovery/goflags v0.1.67/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g=
|
||||
github.com/projectdiscovery/gologger v1.1.41 h1:s5iAOhYDoKnUay5CtvhFpZdGzQKXeARJbpQubThT1Bs=
|
||||
github.com/projectdiscovery/gologger v1.1.41/go.mod h1:68ltkt+p8WquBsi4CQ2Z2nfrDXhuraC7GQbtH15B0PI=
|
||||
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
|
||||
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
|
||||
github.com/projectdiscovery/hmap v0.0.70 h1:1TtvmzJNntKbU9CJI7W5auchg62lzgFYaG4BljwrS1o=
|
||||
github.com/projectdiscovery/hmap v0.0.70/go.mod h1:sugZdHVusZTc45CKtZw7y460C7+JS5SY6ODwlE1i8tI=
|
||||
github.com/projectdiscovery/hmap v0.0.76 h1:m13e18HEJYkCddcnHucn0XxO/tG8Av6TX+U1dw+PQzQ=
|
||||
github.com/projectdiscovery/hmap v0.0.76/go.mod h1:AmLVOcAQKX4Iq4vTE/07LONuctMNUOveofI5EHb4oMQ=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
||||
github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
|
||||
github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU=
|
||||
github.com/projectdiscovery/ratelimit v0.0.54 h1:VwrFLPSfWle5Hg7AvwQkDd4EWDzYWTzd2mDTXTrqvmA=
|
||||
github.com/projectdiscovery/ratelimit v0.0.54/go.mod h1:yGIqMaT8vKG+4mlCqkWOWAvBtvWUlQeanZae31E09cY=
|
||||
github.com/projectdiscovery/rawhttp v0.1.68 h1:6oXatkSkO9a8ybtbhgowQ5wXG0QxmH8a/Ulk4DRsLMM=
|
||||
github.com/projectdiscovery/rawhttp v0.1.68/go.mod h1:gEF17M4VdXbU9Gn9sZXOY75/bXSduq6XuvZTGxw2/2A=
|
||||
github.com/projectdiscovery/retryabledns v1.0.87 h1:MPEXVKdu89FEW23xIMpBzzvdegvtcAs7osSqHimBVOs=
|
||||
github.com/projectdiscovery/retryabledns v1.0.87/go.mod h1:snDTjRcmBj+iveber/o0jC0iLEkM6c0Sdo2IXe2O+fE=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M=
|
||||
github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4=
|
||||
github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw=
|
||||
github.com/projectdiscovery/rawhttp v0.1.84 h1:91ctBCABSR2dIMj+hoQz3E0EsmPDNv2YmsjMOwjyJnk=
|
||||
github.com/projectdiscovery/rawhttp v0.1.84/go.mod h1:F3mb+qlkh/pI+CiaoddmMmgeD2TSPM7WUMcm9DU9KBY=
|
||||
github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0=
|
||||
github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.96 h1:/88LMgEGVx5rpqdLYIqxyJHBluuREKxJxPVIWi1zOlA=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.96/go.mod h1:YIN8SpckCijt5J1AV+nQApUWyCC3Y+8nAwDfK/RwhWs=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
|
||||
github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM=
|
||||
github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM=
|
||||
github.com/projectdiscovery/useragent v0.0.69 h1:6TkVMKjOBDqecUBBkXlTlBUfAUM8Bgn/58E3ZasndBs=
|
||||
github.com/projectdiscovery/useragent v0.0.69/go.mod h1:Ss+Kf6XEjSQqhm7LmbEtOxKvGnnnR88Fx+0PG2PXuBY=
|
||||
github.com/projectdiscovery/utils v0.3.0 h1:JNRMUPMx2gSzdk3bOQPtXSucoz+qC1JVkibIn7+kPTk=
|
||||
github.com/projectdiscovery/utils v0.3.0/go.mod h1:k2XlmfaYO4k6T4vAyUa3Kn/0BxPTIlNiBFpM6nVCbz0=
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.22 h1:7QnpeZy/txsvVcQNlL1n4xIu8IXgMxEsda3w6k6bShY=
|
||||
github.com/projectdiscovery/wappalyzergo v0.1.22/go.mod h1:wnvmbC10pQTOoCKnCTmWKP20rpEtqrMJZvzuTuleeyw=
|
||||
github.com/projectdiscovery/tlsx v1.1.8 h1:Y+VkOp6JmUBb4tci1Fbz9U7ELEQ2irFhm+HS58tHruM=
|
||||
github.com/projectdiscovery/tlsx v1.1.8/go.mod h1:6u/dbLuMsLzmux58AWnAB24qh2+Trk0auCK2I9B17Vo=
|
||||
github.com/projectdiscovery/useragent v0.0.87 h1:XkFxYqwvFaA7Aw3/tnDmXyMuQpBXplTrm0GvrM+aaHY=
|
||||
github.com/projectdiscovery/useragent v0.0.87/go.mod h1:lsKq3LoUWXnn0+Iir8hv+vS3zfMOrKn/aNb33Ik/9CI=
|
||||
github.com/projectdiscovery/utils v0.4.8 h1:/Xd38fP8xc6kifZayjrhcYALenJrjO3sHO7lg+I8ZGk=
|
||||
github.com/projectdiscovery/utils v0.4.8/go.mod h1:S314NzLcXVCbLbwYCoorAJYcnZEwv7Uhw2d3aF5fJ4s=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.12 h1:A3oBpnEbTHOa3Q9m4w/5LLXsmCEiu0mJcwyjf3M9xnc=
|
||||
github.com/projectdiscovery/wappalyzergo v0.2.12/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw=
|
||||
github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
|
||||
github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
@@ -330,8 +330,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
|
||||
@@ -423,8 +423,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
|
||||
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
|
||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
@@ -453,8 +453,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
|
||||
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
|
||||
@@ -496,8 +496,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
|
||||
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -508,8 +508,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
|
||||
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ const banner = `
|
||||
`
|
||||
|
||||
// Version is the current Version of httpx
|
||||
const Version = `v1.6.9`
|
||||
const Version = `v1.6.10`
|
||||
|
||||
// showBanner is used to show the banner to the user
|
||||
func showBanner() {
|
||||
|
||||
+15
-12
@@ -165,18 +165,21 @@ func (s *ScanOptions) Clone() *ScanOptions {
|
||||
|
||||
// Options contains configuration options for httpx.
|
||||
type Options struct {
|
||||
CustomHeaders customheader.CustomHeaders
|
||||
CustomPorts customport.CustomPorts
|
||||
matchStatusCode []int
|
||||
matchContentLength []int
|
||||
filterStatusCode []int
|
||||
filterContentLength []int
|
||||
Output string
|
||||
OutputAll bool
|
||||
StoreResponseDir string
|
||||
OmitBody bool
|
||||
HTTPProxy string
|
||||
CustomHeaders customheader.CustomHeaders
|
||||
CustomPorts customport.CustomPorts
|
||||
matchStatusCode []int
|
||||
matchContentLength []int
|
||||
filterStatusCode []int
|
||||
filterContentLength []int
|
||||
Output string
|
||||
OutputAll bool
|
||||
StoreResponseDir string
|
||||
OmitBody bool
|
||||
// Deprecated: use Proxy
|
||||
HTTPProxy string
|
||||
// Deprecated: use Proxy
|
||||
SocksProxy string
|
||||
Proxy string
|
||||
InputFile string
|
||||
InputTargetHost goflags.StringSlice
|
||||
Methods string
|
||||
@@ -474,7 +477,7 @@ func ParseOptions() *Options {
|
||||
flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"),
|
||||
flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"),
|
||||
flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"),
|
||||
flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"),
|
||||
flagSet.StringVarP(&options.Proxy, "proxy", "http-proxy", "", "proxy (http|socks) to use (eg http://127.0.0.1:8080)"),
|
||||
flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"),
|
||||
flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"),
|
||||
flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"),
|
||||
|
||||
+84
-80
@@ -151,7 +151,12 @@ func New(options *Options) (*Runner, error) {
|
||||
httpxOptions.FollowHostRedirects = options.FollowHostRedirects
|
||||
httpxOptions.RespectHSTS = options.RespectHSTS
|
||||
httpxOptions.MaxRedirects = options.MaxRedirects
|
||||
httpxOptions.HTTPProxy = options.HTTPProxy
|
||||
if options.HTTPProxy != "" {
|
||||
options.Proxy = options.HTTPProxy
|
||||
} else if options.SocksProxy != "" {
|
||||
options.Proxy = options.SocksProxy
|
||||
}
|
||||
httpxOptions.Proxy = options.Proxy
|
||||
httpxOptions.Unsafe = options.Unsafe
|
||||
httpxOptions.UnsafeURI = options.RequestURI
|
||||
httpxOptions.CdnCheck = options.OutputCDN
|
||||
@@ -293,6 +298,7 @@ func New(options *Options) (*Runner, error) {
|
||||
scanopts.NoHeadlessBody = options.NoHeadlessBody
|
||||
scanopts.UseInstalledChrome = options.UseInstalledChrome
|
||||
scanopts.ScreenshotTimeout = options.ScreenshotTimeout
|
||||
scanopts.ScreenshotIdle = options.ScreenshotIdle
|
||||
|
||||
if options.OutputExtractRegexs != nil {
|
||||
for _, regex := range options.OutputExtractRegexs {
|
||||
@@ -1051,94 +1057,92 @@ func (r *Runner) RunEnumeration() {
|
||||
gologger.Silent().Msgf("%s\n", resp.str)
|
||||
}
|
||||
|
||||
if resp.Err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// store responses or chain in directory
|
||||
URL, _ := urlutil.Parse(resp.URL)
|
||||
domainFile := resp.Method + ":" + URL.EscapedString()
|
||||
hash := hashes.Sha1([]byte(domainFile))
|
||||
domainResponseFile := fmt.Sprintf("%s.txt", hash)
|
||||
screenshotResponseFile := fmt.Sprintf("%s.png", hash)
|
||||
hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
|
||||
domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response")
|
||||
domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot")
|
||||
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
|
||||
screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename)
|
||||
if resp.Err == nil {
|
||||
URL, _ := urlutil.Parse(resp.URL)
|
||||
domainFile := resp.Method + ":" + URL.EscapedString()
|
||||
hash := hashes.Sha1([]byte(domainFile))
|
||||
domainResponseFile := fmt.Sprintf("%s.txt", hash)
|
||||
screenshotResponseFile := fmt.Sprintf("%s.png", hash)
|
||||
hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
|
||||
domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response")
|
||||
domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot")
|
||||
responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
|
||||
screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename)
|
||||
|
||||
var responsePath, screenshotPath, screenshotPathRel string
|
||||
// store response
|
||||
if r.scanopts.StoreResponse || r.scanopts.StoreChain {
|
||||
if r.scanopts.OmitBody {
|
||||
resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1)
|
||||
var responsePath, screenshotPath, screenshotPathRel string
|
||||
// store response
|
||||
if r.scanopts.StoreResponse || r.scanopts.StoreChain {
|
||||
if r.scanopts.OmitBody {
|
||||
resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1)
|
||||
}
|
||||
|
||||
responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
|
||||
// URL.EscapedString returns that can be used as filename
|
||||
respRaw := resp.Raw
|
||||
reqRaw := resp.RequestRaw
|
||||
if len(respRaw) > r.scanopts.MaxResponseBodySizeToSave {
|
||||
respRaw = respRaw[:r.scanopts.MaxResponseBodySizeToSave]
|
||||
}
|
||||
data := reqRaw
|
||||
if r.options.StoreChain && resp.Response != nil && resp.Response.HasChain() {
|
||||
data = append(data, append([]byte("\n"), []byte(resp.Response.GetChain())...)...)
|
||||
}
|
||||
data = append(data, respRaw...)
|
||||
data = append(data, []byte("\n\n\n")...)
|
||||
data = append(data, []byte(resp.URL)...)
|
||||
_ = fileutil.CreateFolder(responseBaseDir)
|
||||
writeErr := os.WriteFile(responsePath, data, 0644)
|
||||
if writeErr != nil {
|
||||
gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
|
||||
}
|
||||
resp.StoredResponsePath = responsePath
|
||||
}
|
||||
|
||||
responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
|
||||
// URL.EscapedString returns that can be used as filename
|
||||
respRaw := resp.Raw
|
||||
reqRaw := resp.RequestRaw
|
||||
if len(respRaw) > r.scanopts.MaxResponseBodySizeToSave {
|
||||
respRaw = respRaw[:r.scanopts.MaxResponseBodySizeToSave]
|
||||
}
|
||||
data := reqRaw
|
||||
if r.options.StoreChain && resp.Response != nil && resp.Response.HasChain() {
|
||||
data = append(data, append([]byte("\n"), []byte(resp.Response.GetChain())...)...)
|
||||
}
|
||||
data = append(data, respRaw...)
|
||||
data = append(data, []byte("\n\n\n")...)
|
||||
data = append(data, []byte(resp.URL)...)
|
||||
_ = fileutil.CreateFolder(responseBaseDir)
|
||||
writeErr := os.WriteFile(responsePath, data, 0644)
|
||||
if writeErr != nil {
|
||||
gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
|
||||
}
|
||||
resp.StoredResponsePath = responsePath
|
||||
}
|
||||
if r.scanopts.Screenshot {
|
||||
screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile))
|
||||
screenshotPathRel = filepath.Join(hostFilename, screenshotResponseFile)
|
||||
_ = fileutil.CreateFolder(screenshotBaseDir)
|
||||
err := os.WriteFile(screenshotPath, resp.ScreenshotBytes, 0644)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err)
|
||||
}
|
||||
|
||||
if r.scanopts.Screenshot {
|
||||
screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile))
|
||||
screenshotPathRel = filepath.Join(hostFilename, screenshotResponseFile)
|
||||
_ = fileutil.CreateFolder(screenshotBaseDir)
|
||||
err := os.WriteFile(screenshotPath, resp.ScreenshotBytes, 0644)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not write screenshot at path '%s', to disk: %s", screenshotPath, err)
|
||||
}
|
||||
|
||||
resp.ScreenshotPath = screenshotPath
|
||||
resp.ScreenshotPathRel = screenshotPathRel
|
||||
if r.scanopts.NoScreenshotBytes {
|
||||
resp.ScreenshotBytes = []byte{}
|
||||
}
|
||||
}
|
||||
|
||||
if indexFile != nil {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexFile.WriteString(indexData)
|
||||
}
|
||||
if indexScreenshotFile != nil && resp.ScreenshotPathRel != "" {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPathRel, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexScreenshotFile.WriteString(indexData)
|
||||
}
|
||||
|
||||
if r.scanopts.StoreVisionReconClusters {
|
||||
foundCluster := false
|
||||
pHash, _ := resp.KnowledgeBase["pHash"].(uint64)
|
||||
for i, cluster := range r.pHashClusters {
|
||||
distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash))
|
||||
if distance <= hammingDistanceThreshold {
|
||||
r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL})
|
||||
foundCluster = true
|
||||
break
|
||||
resp.ScreenshotPath = screenshotPath
|
||||
resp.ScreenshotPathRel = screenshotPathRel
|
||||
if r.scanopts.NoScreenshotBytes {
|
||||
resp.ScreenshotBytes = []byte{}
|
||||
}
|
||||
}
|
||||
|
||||
if !foundCluster {
|
||||
newCluster := pHashCluster{
|
||||
BasePHash: pHash,
|
||||
Hashes: []pHashUrl{{PHash: pHash, Url: resp.URL}},
|
||||
if indexFile != nil {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexFile.WriteString(indexData)
|
||||
}
|
||||
if indexScreenshotFile != nil && resp.ScreenshotPathRel != "" {
|
||||
indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPathRel, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
|
||||
_, _ = indexScreenshotFile.WriteString(indexData)
|
||||
}
|
||||
|
||||
if r.scanopts.StoreVisionReconClusters {
|
||||
foundCluster := false
|
||||
pHash, _ := resp.KnowledgeBase["pHash"].(uint64)
|
||||
for i, cluster := range r.pHashClusters {
|
||||
distance, _ := goimagehash.NewImageHash(pHash, goimagehash.PHash).Distance(goimagehash.NewImageHash(cluster.BasePHash, goimagehash.PHash))
|
||||
if distance <= hammingDistanceThreshold {
|
||||
r.pHashClusters[i].Hashes = append(r.pHashClusters[i].Hashes, pHashUrl{PHash: pHash, Url: resp.URL})
|
||||
foundCluster = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !foundCluster {
|
||||
newCluster := pHashCluster{
|
||||
BasePHash: pHash,
|
||||
Hashes: []pHashUrl{{PHash: pHash, Url: resp.URL}},
|
||||
}
|
||||
r.pHashClusters = append(r.pHashClusters, newCluster)
|
||||
}
|
||||
r.pHashClusters = append(r.pHashClusters, newCluster)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -53,20 +53,20 @@ type Result struct {
|
||||
Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
|
||||
Error string `json:"error,omitempty" csv:"error" mapstructure:"error"`
|
||||
WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
|
||||
ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"`
|
||||
ResponseBody string `json:"body,omitempty" csv:"-" mapstructure:"body"`
|
||||
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
|
||||
ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
|
||||
Method string `json:"method,omitempty" csv:"method" mapstructure:"method"`
|
||||
Host string `json:"host,omitempty" csv:"host" mapstructure:"host"`
|
||||
Path string `json:"path,omitempty" csv:"path" mapstructure:"path"`
|
||||
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
|
||||
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"`
|
||||
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"`
|
||||
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
|
||||
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
|
||||
FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
|
||||
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"`
|
||||
RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"`
|
||||
Request string `json:"request,omitempty" csv:"request" mapstructure:"request"`
|
||||
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"-" mapstructure:"header"`
|
||||
RawHeaders string `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"`
|
||||
Request string `json:"request,omitempty" csv:"-" mapstructure:"request"`
|
||||
ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"`
|
||||
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
|
||||
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
|
||||
|
||||
@@ -120,7 +120,7 @@
|
||||
</td>
|
||||
<td style="padding: 10px; border: 1px solid black">
|
||||
<a href="{{.ScreenshotPathRel | safeURL}}">
|
||||
<img src="{{.ScreenshotPathRel | safeURL}}" alt="Screenshot" style="width: 400px; height: 300px" />
|
||||
<img src="{{.ScreenshotPathRel | safeURL}}" alt="Screenshot" style="width: 400px; height: 300px" loading="lazy" />
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
Reference in New Issue
Block a user