mirror of
https://github.com/projectdiscovery/subfinder
synced 2026-06-21 14:05:24 +00:00
Merge branch 'projectdiscovery:dev' into fix/timeout-flag-not-working
This commit is contained in:
Recep Gunes
@@ -0,0 +1,16 @@
|
||||
## Proposed changes
|
||||
|
||||
<!-- Describe the overall picture of your modifications to help maintainers understand the pull request. PRs are required to be associated to their related issue tickets or feature request. -->
|
||||
|
||||
### Proof
|
||||
|
||||
<!-- How has this been tested? Please describe the tests that you ran to verify your changes. -->
|
||||
|
||||
## Checklist
|
||||
|
||||
<!-- Put an "x" in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code. -->
|
||||
|
||||
- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch
|
||||
- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes
|
||||
- [ ] I have added tests that prove my fix is effective or that my feature works
|
||||
- [ ] I have added necessary documentation (if appropriate)
|
||||
@@ -8,7 +8,7 @@ updates:
|
||||
|
||||
# Maintain dependencies for go modules
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "v2/"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
target-branch: "dev"
|
||||
@@ -45,4 +45,4 @@ updates:
|
||||
# prefix: "chore"
|
||||
# include: "scope"
|
||||
# labels:
|
||||
# - "Type: Maintenance"
|
||||
# - "Type: Maintenance"
|
||||
|
||||
@@ -65,6 +65,7 @@ jobs:
|
||||
SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}}
|
||||
SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
|
||||
THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}}
|
||||
URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}}
|
||||
VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}}
|
||||
WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}}
|
||||
ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}}
|
||||
|
||||
@@ -10,14 +10,14 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/projectdiscovery/chaos-client v0.5.2
|
||||
github.com/projectdiscovery/dnsx v1.2.2
|
||||
github.com/projectdiscovery/dnsx v1.2.3
|
||||
github.com/projectdiscovery/fdmax v0.0.4
|
||||
github.com/projectdiscovery/gologger v1.1.54
|
||||
github.com/projectdiscovery/ratelimit v0.0.81
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.115
|
||||
github.com/projectdiscovery/utils v0.4.21
|
||||
github.com/projectdiscovery/gologger v1.1.62
|
||||
github.com/projectdiscovery/ratelimit v0.0.82
|
||||
github.com/projectdiscovery/retryablehttp-go v1.1.0
|
||||
github.com/projectdiscovery/utils v0.7.3
|
||||
github.com/rs/xid v1.5.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -49,7 +49,7 @@ require (
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/fatih/color v1.15.0 // indirect
|
||||
github.com/gaissmai/bart v0.20.4 // indirect
|
||||
github.com/gaissmai/bart v0.26.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/go-github/v30 v30.1.0 // indirect
|
||||
@@ -77,12 +77,12 @@ require (
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/projectdiscovery/blackrock v0.0.1 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.1.24 // indirect
|
||||
github.com/projectdiscovery/fastdialer v0.4.1 // indirect
|
||||
github.com/projectdiscovery/hmap v0.0.90 // indirect
|
||||
github.com/projectdiscovery/cdncheck v1.2.13 // indirect
|
||||
github.com/projectdiscovery/fastdialer v0.4.19 // indirect
|
||||
github.com/projectdiscovery/hmap v0.0.98 // indirect
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
|
||||
github.com/projectdiscovery/networkpolicy v0.1.16 // indirect
|
||||
github.com/refraction-networking/utls v1.7.0 // indirect
|
||||
github.com/projectdiscovery/networkpolicy v0.1.31 // indirect
|
||||
github.com/refraction-networking/utls v1.7.1 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
|
||||
@@ -101,7 +101,7 @@ require (
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/weppos/publicsuffix-go v0.30.1 // indirect
|
||||
github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 // indirect
|
||||
github.com/yuin/goldmark v1.7.4 // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.3 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
@@ -132,7 +132,7 @@ require (
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/projectdiscovery/goflags v0.1.74
|
||||
github.com/projectdiscovery/retryabledns v1.0.102 // indirect
|
||||
github.com/projectdiscovery/retryabledns v1.0.111 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
)
|
||||
|
||||
@@ -26,7 +26,6 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub
|
||||
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
|
||||
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
|
||||
github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
|
||||
github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
@@ -59,7 +58,6 @@ github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A
|
||||
github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc=
|
||||
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
|
||||
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
|
||||
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
|
||||
github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
|
||||
@@ -75,7 +73,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
|
||||
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
|
||||
@@ -103,8 +100,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
|
||||
github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
|
||||
github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
|
||||
github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
@@ -142,7 +139,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
|
||||
github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
|
||||
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
|
||||
github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA=
|
||||
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
@@ -255,37 +251,37 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
|
||||
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
|
||||
github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs=
|
||||
github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
|
||||
github.com/projectdiscovery/cdncheck v1.2.13 h1:6zs4Mn8JV3yKyMoAr857Hf2NLvyOMpOfqCCT2V2OI1Q=
|
||||
github.com/projectdiscovery/cdncheck v1.2.13/go.mod h1:/OhuZ9T25yXSqU6+oWvmVQ3QFvtew/Tp03u0jM+NJBE=
|
||||
github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU=
|
||||
github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU=
|
||||
github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg=
|
||||
github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I=
|
||||
github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4=
|
||||
github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I=
|
||||
github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc=
|
||||
github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE=
|
||||
github.com/projectdiscovery/fastdialer v0.4.19 h1:MLHwEGM0x0pyltJaNvAVvwc27bnXdZ5mYr50S/2kMEE=
|
||||
github.com/projectdiscovery/fastdialer v0.4.19/go.mod h1:HGdVsez+JgJ9/ljXjHRplOqkB7og+nqi0nrNWVNi03o=
|
||||
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
|
||||
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
|
||||
github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c=
|
||||
github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4=
|
||||
github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws=
|
||||
github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE=
|
||||
github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4=
|
||||
github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY=
|
||||
github.com/projectdiscovery/gologger v1.1.62 h1:wzKqvL6HQRzf0/PpBEhInZqqL1q4mKe2gFGJeDG3FqE=
|
||||
github.com/projectdiscovery/gologger v1.1.62/go.mod h1:YWvMSxlHybU3SkFCcWn+driSJ8yY+3CR3g/textnp+Y=
|
||||
github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg=
|
||||
github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
|
||||
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U=
|
||||
github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI=
|
||||
github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA=
|
||||
github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI=
|
||||
github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA=
|
||||
github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q=
|
||||
github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.31 h1:mE6iJeYOSql8gps/91vwiztE/kEHe5Im8oUO5Mkj9Zg=
|
||||
github.com/projectdiscovery/networkpolicy v0.1.31/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU=
|
||||
github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak=
|
||||
github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM=
|
||||
github.com/projectdiscovery/retryabledns v1.0.111 h1:iyMdCDgNmaSRJYcGqB+SLlvlw9WijlbJ6Q9OEpRAWsQ=
|
||||
github.com/projectdiscovery/retryabledns v1.0.111/go.mod h1:6TOPJ3QAE4reBu6bvsGsTcyEb+OypcKYFQH7yVsjyIM=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.1.0 h1:uYp3EnuhhamTwvG41X6q6TAc/SHEO9pw9CBWbRASIQk=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5XFDTMjy1yKeMyn1kj35lPlcfM=
|
||||
github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs=
|
||||
github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs=
|
||||
github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms=
|
||||
github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0=
|
||||
github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
@@ -322,8 +318,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
@@ -363,8 +359,8 @@ github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
|
||||
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
|
||||
github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs=
|
||||
github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos=
|
||||
github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
|
||||
github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM=
|
||||
github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
|
||||
@@ -414,7 +410,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -470,8 +465,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@@ -480,7 +473,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
|
||||
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -516,7 +508,6 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -534,8 +525,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -548,8 +537,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
||||
@@ -53,6 +53,7 @@ import (
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
|
||||
@@ -114,6 +115,7 @@ var AllSources = [...]subscraping.Source{
|
||||
&hudsonrock.Source{},
|
||||
&digitalyama.Source{},
|
||||
&thc.Source{},
|
||||
&urlscan.Source{},
|
||||
}
|
||||
|
||||
var sourceWarnings = mapsutil.NewSyncLockMap[string, string](
|
||||
|
||||
@@ -63,6 +63,7 @@ var (
|
||||
"digitalyama",
|
||||
"merklemap",
|
||||
"thc",
|
||||
"urlscan",
|
||||
}
|
||||
|
||||
expectedDefaultSources = []string{
|
||||
@@ -105,6 +106,7 @@ var (
|
||||
"builtwith",
|
||||
"digitalyama",
|
||||
"thc",
|
||||
"urlscan",
|
||||
}
|
||||
|
||||
expectedDefaultRecursiveSources = []string{
|
||||
@@ -121,6 +123,7 @@ var (
|
||||
"leakix",
|
||||
"facebook",
|
||||
"merklemap",
|
||||
"urlscan",
|
||||
// "reconcloud",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -34,6 +34,9 @@ func TestSourcesWithoutKeys(t *testing.T) {
|
||||
"dnsdumpster", // failing with "unexpected status code 403 received"
|
||||
"anubis", // failing with "too many redirects"
|
||||
"threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times"
|
||||
"leakix", // now requires API key (returns 401)
|
||||
"reconeer", // now requires API key (returns 401)
|
||||
"sitedossier", // flaky - returns no results in CI
|
||||
}
|
||||
|
||||
domain := "hackerone.com"
|
||||
|
||||
@@ -265,4 +265,5 @@ var defaultRateLimits = []string{
|
||||
// "gitlab=2/s",
|
||||
"github=83/m",
|
||||
"hudsonrock=5/s",
|
||||
"urlscan=1/s",
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
"github.com/projectdiscovery/utils/generic"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
@@ -128,7 +127,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
|
||||
response := &response{}
|
||||
if err := json.Unmarshal(bin, response); err != nil {
|
||||
s.errors++
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))}
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)}
|
||||
return
|
||||
}
|
||||
for _, v := range response.Data {
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// Package urlscan logic
|
||||
package urlscan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
|
||||
)
|
||||
|
||||
const (
|
||||
// baseURL is the URLScan API base URL
|
||||
baseURL = "https://urlscan.io/api/v1/search/"
|
||||
// maxPages is the maximum number of pages to fetch
|
||||
maxPages = 5
|
||||
// maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer)
|
||||
maxPerPage = 100
|
||||
)
|
||||
|
||||
// response represents the URLScan API response structure
|
||||
type response struct {
|
||||
Results []struct {
|
||||
Task struct {
|
||||
Domain string `json:"domain"`
|
||||
URL string `json:"url"`
|
||||
} `json:"task"`
|
||||
Page struct {
|
||||
Domain string `json:"domain"`
|
||||
URL string `json:"url"`
|
||||
} `json:"page"`
|
||||
Sort []interface{} `json:"sort"`
|
||||
} `json:"results"`
|
||||
HasMore bool `json:"has_more"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// Source is the passive scraping agent
|
||||
type Source struct {
|
||||
apiKeys []string
|
||||
timeTaken time.Duration
|
||||
errors int
|
||||
results int
|
||||
requests int
|
||||
skipped bool
|
||||
}
|
||||
|
||||
// Run function returns all subdomains found with the service
|
||||
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
|
||||
results := make(chan subscraping.Result)
|
||||
s.errors = 0
|
||||
s.results = 0
|
||||
s.requests = 0
|
||||
s.skipped = false
|
||||
|
||||
go func() {
|
||||
defer func(startTime time.Time) {
|
||||
s.timeTaken = time.Since(startTime)
|
||||
close(results)
|
||||
}(time.Now())
|
||||
|
||||
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
|
||||
if randomApiKey == "" {
|
||||
s.skipped = true
|
||||
return
|
||||
}
|
||||
|
||||
headers := map[string]string{"api-key": randomApiKey}
|
||||
|
||||
// Search with wildcard to get more subdomain results
|
||||
s.enumerate(ctx, domain, headers, session, results)
|
||||
}()
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// enumerate performs the actual enumeration with pagination
|
||||
func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) {
|
||||
var searchAfter string
|
||||
currentPage := 0
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if currentPage >= maxPages {
|
||||
break
|
||||
}
|
||||
|
||||
// Build search URL
|
||||
searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage)
|
||||
if searchAfter != "" {
|
||||
searchURL += "&search_after=" + url.QueryEscape(searchAfter)
|
||||
}
|
||||
|
||||
s.requests++
|
||||
resp, err := session.Get(ctx, searchURL, "", headers)
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
s.errors++
|
||||
session.DiscardHTTPResponse(resp)
|
||||
return
|
||||
}
|
||||
|
||||
var data response
|
||||
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
|
||||
if err != nil {
|
||||
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
|
||||
s.errors++
|
||||
session.DiscardHTTPResponse(resp)
|
||||
return
|
||||
}
|
||||
session.DiscardHTTPResponse(resp)
|
||||
|
||||
// Process results - extract subdomains from multiple fields
|
||||
for _, result := range data.Results {
|
||||
candidates := []string{
|
||||
result.Task.Domain,
|
||||
result.Page.Domain,
|
||||
}
|
||||
|
||||
// Also extract from URLs if present
|
||||
if result.Task.URL != "" {
|
||||
if u, err := url.Parse(result.Task.URL); err == nil {
|
||||
candidates = append(candidates, u.Hostname())
|
||||
}
|
||||
}
|
||||
if result.Page.URL != "" {
|
||||
if u, err := url.Parse(result.Page.URL); err == nil {
|
||||
candidates = append(candidates, u.Hostname())
|
||||
}
|
||||
}
|
||||
|
||||
for _, candidate := range candidates {
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
for _, subdomain := range session.Extractor.Extract(candidate) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
|
||||
s.results++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check pagination conditions
|
||||
if !data.HasMore || len(data.Results) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Get sort value for next page
|
||||
lastResult := data.Results[len(data.Results)-1]
|
||||
if len(lastResult.Sort) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Build search_after parameter
|
||||
sortValues := make([]string, len(lastResult.Sort))
|
||||
for i, v := range lastResult.Sort {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
sortValues[i] = fmt.Sprintf("%.0f", val)
|
||||
default:
|
||||
sortValues[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
searchAfter = strings.Join(sortValues, ",")
|
||||
currentPage++
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the source
|
||||
func (s *Source) Name() string {
|
||||
return "urlscan"
|
||||
}
|
||||
|
||||
func (s *Source) IsDefault() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Source) HasRecursiveSupport() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
|
||||
return subscraping.RequiredKey
|
||||
}
|
||||
|
||||
func (s *Source) NeedsKey() bool {
|
||||
return s.KeyRequirement() == subscraping.RequiredKey
|
||||
}
|
||||
|
||||
func (s *Source) AddApiKeys(keys []string) {
|
||||
s.apiKeys = keys
|
||||
}
|
||||
|
||||
func (s *Source) Statistics() subscraping.Statistics {
|
||||
return subscraping.Statistics{
|
||||
Errors: s.errors,
|
||||
Results: s.results,
|
||||
Requests: s.requests,
|
||||
TimeTaken: s.timeTaken,
|
||||
Skipped: s.skipped,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user