admin/projectdiscovery-subfinder
1
0
Fork 0
mirror of https://github.com/projectdiscovery/subfinder synced 2026-06-21 14:05:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of github.com:x-stp/subfinder into feat/submd-dev

Browse Source 
# Conflicts:
#	pkg/passive/sources.go
#	pkg/passive/sources_test.go
This commit is contained in:
x-stp
2026-04-09 09:29:37 +02:00
parent ae1c9a6e32 85929ab9ee
commit c64aa32dbd
89 changed files with 3151 additions and 988 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/PULL_REQUEST_TEMPLATE.md
+16
View File
@@ -0,0 +1,16 @@
## Proposed changes
<!-- Describe the overall picture of your modifications to help maintainers understand the pull request. PRs are required to be associated to their related issue tickets or feature request. -->
### Proof
<!-- How has this been tested? Please describe the tests that you ran to verify your changes. -->
## Checklist
<!-- Put an "x" in the boxes that apply. You can also fill these out after creating the PR. If you're unsure about any of them, don't hesitate to ask. We're here to help! This is simply a reminder of what we are going to look for before merging your code. -->
- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch
- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] I have added necessary documentation (if appropriate)
.github/dependabot.yml
+2 -2
View File
@@ -8,7 +8,7 @@ updates:
# Maintain dependencies for go modules
- package-ecosystem: "gomod"
directory: "v2/"
directory: "/"
schedule:
interval: "weekly"
target-branch: "dev"
@@ -45,4 +45,4 @@ updates:
# prefix: "chore"
# include: "scope"
# labels:
# - "Type: Maintenance"
# - "Type: Maintenance"
.github/release.yml
+2 -2
View File
@@ -8,10 +8,10 @@ changelog:
- "Type: Enhancement"
- title: 🐞 Bug Fixes
labels:
- "Type: Bug"
- "Type: Bug"
- title: 🔨 Maintenance
labels:
- "Type: Maintenance"
- "Type: Maintenance"
- title: Other Changes
labels:
- "*"
.github/workflows/build-test.yml
+7 -14
View File
@@ -19,15 +19,9 @@ jobs:
if: "${{ !endsWith(github.actor, '[bot]') }}"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
with:
go-version-file: go.mod
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v8
with:
version: latest
args: --timeout 5m
- uses: projectdiscovery/actions/golangci-lint/v2@v1
build:
name: Test Builds
@@ -35,12 +29,10 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-13]
os: [ubuntu-latest, windows-latest, macOS-latest]
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go@v1
with:
go-version-file: go.mod
- run: go build ./...
- name: Run tests
@@ -58,7 +50,6 @@ jobs:
FOFA_API_KEY: ${{secrets.FOFA_API_KEY}}
FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}}
GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}}
HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}}
INTELX_API_KEY: ${{secrets.INTELX_API_KEY}}
LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}}
QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}}
@@ -66,10 +57,11 @@ jobs:
SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}}
SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}}
URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}}
VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}}
WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}}
ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}}
uses: nick-invision/retry@v2
uses: nick-fields/retry@v3
with:
timeout_seconds: 360
max_attempts: 3
@@ -80,3 +72,4 @@ jobs:
- name: Run Example
run: go run .
working-directory: examples
.github/workflows/codeql-analysis.yml
+16 -15
View File
@@ -7,14 +7,15 @@ on:
- '**.go'
- '**.mod'
permissions:
actions: read
contents: read
security-events: write
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
@@ -23,17 +24,17 @@ jobs:
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Checkout repository
uses: actions/checkout@v6
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v2
- name: Autobuild
uses: github/codeql-action/autobuild@v4
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
.github/workflows/compat-checks.yaml → .github/workflows/compat-checks.yml
+6 -6
View File
@@ -6,15 +6,15 @@ on:
branches:
- dev
permissions:
contents: write
jobs:
check:
if: github.actor == 'dependabot[bot]'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: projectdiscovery/actions/setup/go/compat-checks@master
- uses: actions/checkout@v6
- uses: projectdiscovery/actions/setup/go/compat-checks@v1
with:
go-version-file: 'go.mod'
go-version-file: go.mod
.github/workflows/dep-auto-merge.yml
+2 -2
View File
@@ -16,11 +16,11 @@ jobs:
runs-on: ubuntu-latest
if: github.actor == 'dependabot[bot]'
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v6
with:
token: ${{ secrets.DEPENDABOT_PAT }}
- uses: ahmadnassri/action-dependabot-auto-merge@v2
with:
github-token: ${{ secrets.DEPENDABOT_PAT }}
target: all
target: all
.github/workflows/dockerhub-push.yml
+5 -5
View File
@@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest-16-cores
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v6
- name: Get Github tag
id: meta
@@ -20,19 +20,19 @@ jobs:
curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64,linux/arm
.github/workflows/release-binary.yml
+9 -13
View File
@@ -11,20 +11,16 @@ jobs:
runs-on: ubuntu-latest-16-cores
steps:
- name: "Check out code"
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v4
with:
go-version: 1.21.x
- name: "Create release on GitHub"
uses: goreleaser/goreleaser-action@v3
uses: actions/checkout@v6
with:
args: "release --clean"
version: latest
fetch-depth: 0
- uses: projectdiscovery/actions/setup/go@v1
- name: "Create release on GitHub"
uses: projectdiscovery/actions/goreleaser@v1
with:
release: true
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
.github/workflows/release-test.yml
+5 -11
View File
@@ -12,17 +12,11 @@ jobs:
runs-on: ubuntu-latest-16-cores
steps:
- name: "Check out code"
uses: actions/checkout@v3
with:
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.21.x
- uses: projectdiscovery/actions/setup/go@v1
- name: release test
uses: goreleaser/goreleaser-action@v4
with:
args: "release --clean --snapshot"
version: latest
uses: projectdiscovery/actions/goreleaser@v1
.gitignore
+2 -1
View File
@@ -8,4 +8,5 @@ vendor/
.idea
.devcontainer
.vscode
dist
dist
/subfinder
.goreleaser.yml
+21 -21
View File
@@ -5,31 +5,31 @@ before:
- go mod tidy
builds:
- env:
- CGO_ENABLED=0
goos:
- windows
- linux
- darwin
goarch:
- amd64
- 386
- arm
- arm64
- env:
- CGO_ENABLED=0
goos:
- windows
- linux
- darwin
goarch:
- amd64
- '386'
- arm
- arm64
ignore:
- goos: darwin
goarch: '386'
- goos: windows
goarch: 'arm'
ignore:
- goos: darwin
goarch: '386'
- goos: windows
goarch: 'arm'
binary: '{{ .ProjectName }}'
main: cmd/subfinder/main.go
binary: '{{ .ProjectName }}'
main: cmd/subfinder/main.go
archives:
- formats:
- zip
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
- formats:
- zip
name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}'
checksum:
algorithm: sha256
README.md
+7
View File
@@ -109,6 +109,13 @@ OPTIMIZATION:
-max-time int minutes to wait for enumeration results (default 10)
```
## Environment Variables
Subfinder supports environment variables to specify custom paths for configuration files:
- `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`)
- `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`)
# Installation
`subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version:
go.mod
+38 -38
View File
@@ -10,14 +10,13 @@ require (
github.com/json-iterator/go v1.1.12
github.com/lib/pq v1.10.9
github.com/projectdiscovery/chaos-client v0.5.2
github.com/projectdiscovery/dnsx v1.2.2
github.com/projectdiscovery/dnsx v1.2.3
github.com/projectdiscovery/fdmax v0.0.4
github.com/projectdiscovery/gologger v1.1.54
github.com/projectdiscovery/ratelimit v0.0.81
github.com/projectdiscovery/retryablehttp-go v1.0.115
github.com/projectdiscovery/utils v0.4.21
github.com/projectdiscovery/gologger v1.1.68
github.com/projectdiscovery/ratelimit v0.0.84
github.com/projectdiscovery/utils v0.10.0
github.com/rs/xid v1.5.0
github.com/stretchr/testify v1.10.0
github.com/stretchr/testify v1.11.1
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8
gopkg.in/yaml.v3 v3.0.1
@@ -28,28 +27,28 @@ require (
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/STARRY-S/zip v0.2.1 // indirect
github.com/STARRY-S/zip v0.2.3 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.6.0 // indirect
github.com/bodgit/sevenzip v1.6.1 // indirect
github.com/bodgit/windows v1.0.1 // indirect
github.com/charmbracelet/glamour v0.8.0 // indirect
github.com/charmbracelet/lipgloss v0.13.0 // indirect
github.com/charmbracelet/x/ansi v0.3.2 // indirect
github.com/cheggaaa/pb/v3 v3.1.4 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/djherbis/times v1.6.0 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/gaissmai/bart v0.20.4 // indirect
github.com/gaissmai/bart v0.26.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
@@ -57,39 +56,41 @@ require (
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mholt/archives v0.1.0 // indirect
github.com/mholt/archives v0.1.5 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/nwaples/rardecode/v2 v2.2.2 // indirect
github.com/pierrec/lz4/v4 v4.1.23 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/cdncheck v1.1.24 // indirect
github.com/projectdiscovery/fastdialer v0.4.1 // indirect
github.com/projectdiscovery/hmap v0.0.90 // indirect
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
github.com/projectdiscovery/networkpolicy v0.1.16 // indirect
github.com/refraction-networking/utls v1.7.0 // indirect
github.com/projectdiscovery/cdncheck v1.2.28 // indirect
github.com/projectdiscovery/fastdialer v0.5.4 // indirect
github.com/projectdiscovery/hmap v0.0.100 // indirect
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
github.com/projectdiscovery/networkpolicy v0.1.34 // indirect
github.com/projectdiscovery/retryablehttp-go v1.3.6 // indirect
github.com/refraction-networking/utls v1.8.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sorairolake/lzip-go v0.3.5 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tidwall/btree v1.6.0 // indirect
github.com/tidwall/buntdb v1.3.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
@@ -100,26 +101,25 @@ require (
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/weppos/publicsuffix-go v0.30.1 // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 // indirect
github.com/yuin/goldmark v1.7.4 // indirect
github.com/yuin/goldmark-emoji v1.0.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zcalusic/sysinfo v1.0.2 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.uber.org/multierr v1.11.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/mod v0.22.0 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/oauth2 v0.27.0 // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.29.0 // indirect
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
golang.org/x/tools v0.39.0 // indirect
)
require (
@@ -132,7 +132,7 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/projectdiscovery/goflags v0.1.74
github.com/projectdiscovery/retryabledns v1.0.102 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.31.0 // indirect
github.com/projectdiscovery/retryabledns v1.0.114 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/sys v0.39.0 // indirect
)
go.sum
+113 -89
View File
@@ -27,8 +27,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2piv
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
@@ -39,8 +39,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46
github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
@@ -55,8 +55,8 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk
github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A=
github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc=
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8=
github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4=
github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -76,8 +76,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE=
@@ -89,6 +87,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -103,8 +103,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U=
github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk=
github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -137,11 +137,11 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA=
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -162,11 +162,6 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw=
github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -183,8 +178,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -200,6 +195,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -212,12 +209,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q=
github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A=
github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -231,8 +232,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg=
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ=
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY=
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU=
github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -243,8 +244,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU=
github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -255,37 +256,37 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs=
github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U=
github.com/projectdiscovery/cdncheck v1.2.28 h1:EhOK+mi0bfLb+CdiJaFYmMMuX7xu0U820e5GFbP2usc=
github.com/projectdiscovery/cdncheck v1.2.28/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I=
github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU=
github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU=
github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg=
github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I=
github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4=
github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I=
github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc=
github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE=
github.com/projectdiscovery/fastdialer v0.5.4 h1:+0oesDDqZcIPE5bNDmm/Xm9Xm3yjnhl4xwP+h5D1TE4=
github.com/projectdiscovery/fastdialer v0.5.4/go.mod h1:KCzt6WnSAj9umiUBRCaC0EJSEyeshxDoowfwjxodmQw=
github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c=
github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4=
github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws=
github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE=
github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4=
github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI=
github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U=
github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI=
github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA=
github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI=
github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E=
github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E=
github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA=
github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q=
github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I=
github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU=
github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o=
github.com/projectdiscovery/hmap v0.0.100 h1:DBZ3Req9lWf4P1YC9PRa4eiMvLY0Uxud43NRBcocPfs=
github.com/projectdiscovery/hmap v0.0.100/go.mod h1:2O06pR8pHOP9wSmxAoxuM45U7E+UqOqOdlSIeddM0bA=
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q=
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
github.com/projectdiscovery/networkpolicy v0.1.34 h1:TRwNbgMwdx3NC190TKSLwtTvr0JAIZAlnWkOhW0yBME=
github.com/projectdiscovery/networkpolicy v0.1.34/go.mod h1:GJ20E7fJoA2vk8ZBSa1Cvc5WyP8RxglF5bZmYgK8jag=
github.com/projectdiscovery/ratelimit v0.0.84 h1:RzaO+OBGABI5G0e045TLwedV7K6o0dTOy9pxYmH+tbU=
github.com/projectdiscovery/ratelimit v0.0.84/go.mod h1:U0OGcBJk6+Hz9nERWZVN4s8XL3pF9XxQJwPlEbF4pr0=
github.com/projectdiscovery/retryabledns v1.0.114 h1:COyNKzhA7oa3C/1639WRXeXsKrUJx06paVbN64IHZ3E=
github.com/projectdiscovery/retryabledns v1.0.114/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA=
github.com/projectdiscovery/retryablehttp-go v1.3.6 h1:dLb0/YVX+oX70gpWxN5GXT8pCKpn8fdXfwOq2TsXxNY=
github.com/projectdiscovery/retryablehttp-go v1.3.6/go.mod h1:tKVxmL4ixWy1MjYk5GJvFL0Cp10fnQgSp2F6bSBEypI=
github.com/projectdiscovery/utils v0.10.0 h1:E3nMm0h3LWt2bbnpRd8Whyj/y0DrMJKYx2zV8gqFDF0=
github.com/projectdiscovery/utils v0.10.0/go.mod h1:FL0cQdg3oBMtJdmbBrfLd5i73syNxpkbKO9tivQ0+rI=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs=
github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -307,13 +308,17 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg=
github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik=
github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -322,12 +327,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
@@ -359,12 +362,12 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs=
github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos=
github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko=
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0=
github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
@@ -387,8 +390,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t
github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk=
github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA=
github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM=
github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
@@ -409,14 +412,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -446,8 +450,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -471,15 +478,17 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
@@ -491,8 +500,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -518,6 +530,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -527,17 +540,25 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -549,9 +570,12 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
@@ -582,8 +606,10 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE=
golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -624,13 +650,11 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o=
gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
pkg/passive/passive.go
+43 -7
View File
@@ -11,6 +11,7 @@ import (
"github.com/projectdiscovery/ratelimit"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
mapsutil "github.com/projectdiscovery/utils/maps"
)
type EnumerationOptions struct {
@@ -65,11 +66,15 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p
for _, runner := range a.sources {
wg.Add(1)
go func(source subscraping.Source) {
defer wg.Done()
ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name())
for resp := range source.Run(ctxWithValue, domain, session) {
results <- resp
select {
case <-ctx.Done():
return
case results <- resp:
}
}
wg.Done()
}(runner)
}
wg.Wait()
@@ -81,14 +86,21 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p
func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) {
var multiRateLimiter *ratelimit.MultiLimiter
var err error
for _, source := range a.sources {
var rl uint
if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok {
rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit)
if rateLimit == nil {
rateLimit = &subscraping.CustomRateLimit{
Custom: mapsutil.SyncLockMap[string, uint]{
Map: make(map[string]uint),
},
CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{
Map: make(map[string]time.Duration),
},
}
}
for _, source := range a.sources {
rl, duration := resolveSourceRateLimit(globalRateLimit, rateLimit, source.Name())
if rl > 0 {
multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second)
multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, duration)
} else {
multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond)
}
@@ -100,6 +112,30 @@ func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int,
return multiRateLimiter, err
}
// resolveSourceRateLimit returns the effective rate limit and duration for a source.
// Priority: per-source custom limit > global -rl limit > unlimited.
// Duration comes from -rls (e.g. hackertarget=2/m → 2 per minute), defaulting to per-second.
func resolveSourceRateLimit(globalRateLimit int, rateLimit *subscraping.CustomRateLimit, sourceName string) (uint, time.Duration) {
duration := time.Second // default: requests per second
sourceLower := strings.ToLower(sourceName)
if sourceRL, ok := rateLimit.Custom.Get(sourceLower); ok {
rl := sourceRateLimitOrDefault(uint(max(globalRateLimit, 0)), sourceRL)
// Use per-source duration from -rls if set (e.g. "2/m" → time.Minute)
if d, ok := rateLimit.CustomDuration.Get(sourceLower); ok && d > 0 {
duration = d
}
return rl, duration
}
// No per-source limit: fall back to global -rl
if globalRateLimit > 0 {
return uint(globalRateLimit), duration
}
return 0, duration
}
func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint {
if sourceRateLimit > 0 {
return sourceRateLimit
pkg/passive/ratelimit_test.go
+147
View File
@@ -0,0 +1,147 @@
package passive
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
mapsutil "github.com/projectdiscovery/utils/maps"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
func newCustomRateLimit(counts map[string]uint, durations map[string]time.Duration) *subscraping.CustomRateLimit {
crl := &subscraping.CustomRateLimit{
Custom: mapsutil.SyncLockMap[string, uint]{
Map: make(map[string]uint),
},
CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{
Map: make(map[string]time.Duration),
},
}
for k, v := range counts {
_ = crl.Custom.Set(k, v)
}
for k, d := range durations {
_ = crl.CustomDuration.Set(k, d)
}
return crl
}
func TestResolveSourceRateLimit_PerSourceOverride(t *testing.T) {
// Per-source limit should override global
crl := newCustomRateLimit(
map[string]uint{"hackertarget": 5},
map[string]time.Duration{"hackertarget": time.Minute},
)
rl, dur := resolveSourceRateLimit(10, crl, "hackertarget")
assert.Equal(t, uint(5), rl)
assert.Equal(t, time.Minute, dur)
}
func TestResolveSourceRateLimit_GlobalFallback(t *testing.T) {
// Sources without per-source limit should use global -rl
crl := newCustomRateLimit(nil, nil)
rl, dur := resolveSourceRateLimit(10, crl, "crtsh")
assert.Equal(t, uint(10), rl)
assert.Equal(t, time.Second, dur)
}
func TestResolveSourceRateLimit_NoLimit(t *testing.T) {
// No global and no per-source → unlimited (0)
crl := newCustomRateLimit(nil, nil)
rl, _ := resolveSourceRateLimit(0, crl, "crtsh")
assert.Equal(t, uint(0), rl)
}
func TestResolveSourceRateLimit_ZeroValueRateLimit(t *testing.T) {
// Zero-value CustomRateLimit (nil maps) should not panic and fall back to global
rl, dur := resolveSourceRateLimit(5, &subscraping.CustomRateLimit{}, "crtsh")
assert.Equal(t, uint(5), rl)
assert.Equal(t, time.Second, dur)
}
func TestResolveSourceRateLimit_PerSourceDefaultDuration(t *testing.T) {
// Per-source limit without explicit duration defaults to per-second
crl := newCustomRateLimit(
map[string]uint{"hackertarget": 3},
nil,
)
rl, dur := resolveSourceRateLimit(0, crl, "hackertarget")
assert.Equal(t, uint(3), rl)
assert.Equal(t, time.Second, dur)
}
func TestBuildMultiRateLimiter_GlobalAppliedToAll(t *testing.T) {
// With -rl 5 and no per-source overrides, all sources should be rate-limited
agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false)
crl := newCustomRateLimit(nil, nil)
limiter, err := agent.buildMultiRateLimiter(context.Background(), 5, crl)
require.NoError(t, err)
require.NotNil(t, limiter)
}
func TestBuildMultiRateLimiter_NilRateLimit(t *testing.T) {
// nil rateLimit should not panic (guard in buildMultiRateLimiter)
agent := New([]string{"hackertarget"}, []string{}, false, false)
limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, nil)
require.NoError(t, err)
require.NotNil(t, limiter)
}
func TestBuildMultiRateLimiter_PerSourceWithDuration(t *testing.T) {
// Per-source limit with custom duration (e.g. -rls hackertarget=2/m)
agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false)
crl := newCustomRateLimit(
map[string]uint{"hackertarget": 2},
map[string]time.Duration{"hackertarget": time.Minute},
)
limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl)
require.NoError(t, err)
require.NotNil(t, limiter)
}
func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) {
// Without any rate limits, sources should get MaxUint32 (effectively unlimited)
agent := New([]string{"hackertarget"}, []string{}, false, false)
crl := newCustomRateLimit(nil, nil)
limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl)
require.NoError(t, err)
require.NotNil(t, limiter)
}
func TestResolveSourceRateLimit_PerSourceExceedsGlobal(t *testing.T) {
// Per-source limit higher than global should be honoured
// (the source was explicitly allowed more than the default)
crl := newCustomRateLimit(
map[string]uint{"hackertarget": 20},
nil,
)
rl, dur := resolveSourceRateLimit(10, crl, "hackertarget")
assert.Equal(t, uint(20), rl, "per-source limit should take precedence even when > global")
assert.Equal(t, time.Second, dur)
}
func TestResolveSourceRateLimit_CaseInsensitive(t *testing.T) {
// Source names should be case-insensitive for lookup
crl := newCustomRateLimit(
map[string]uint{"hackertarget": 5},
nil,
)
// Try mixed case — should still match the lower-cased key
rl, _ := resolveSourceRateLimit(0, crl, "HackerTarget")
assert.Equal(t, uint(5), rl, "source name lookup should be case-insensitive")
}
pkg/passive/sources.go
+25 -13
View File
@@ -26,20 +26,23 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud"
@@ -47,11 +50,14 @@ import (
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
mapsutil "github.com/projectdiscovery/utils/maps"
)
@@ -61,6 +67,7 @@ var AllSources = [...]subscraping.Source{
&anubis.Source{},
&bevigil.Source{},
&bufferover.Source{},
&builtwith.Source{},
&c99.Source{},
&censys.Source{},
&certspotter.Source{},
@@ -68,22 +75,29 @@ var AllSources = [...]subscraping.Source{
&chinaz.Source{},
&commoncrawl.Source{},
&crtsh.Source{},
&digitalyama.Source{},
&digitorus.Source{},
&dnsdb.Source{},
&dnsdumpster.Source{},
&dnsrepo.Source{},
&domainsproject.Source{},
&driftnet.Source{},
&fofa.Source{},
&fullhunt.Source{},
&github.Source{},
&hackertarget.Source{},
&hunter.Source{},
&hudsonrock.Source{},
&intelx.Source{},
&netlas.Source{},
&leakix.Source{},
&quake.Source{},
&merklemap.Source{},
&netlas.Source{},
&onyphe.Source{},
&profundis.Source{},
&pugrecon.Source{},
&quake.Source{},
&rapiddns.Source{},
// &reconcloud.Source{}, // failing due to cloudflare bot protection
&reconeer.Source{},
&redhuntlabs.Source{},
// &riddler.Source{}, // failing due to cloudfront protection
&robtex.Source{},
@@ -91,18 +105,16 @@ var AllSources = [...]subscraping.Source{
&securitytrails.Source{},
&shodan.Source{},
&sitedossier.Source{},
&thc.Source{},
&threatbook.Source{},
&threatcrowd.Source{},
// &threatminer.Source{}, // failing api
&urlscan.Source{},
&virustotal.Source{},
&waybackarchive.Source{},
&whoisxmlapi.Source{},
&windvane.Source{},
&zoomeyeapi.Source{},
&facebook.Source{},
// &threatminer.Source{}, // failing api
// &reconcloud.Source{}, // failing due to cloudflare bot protection
&builtwith.Source{},
&hudsonrock.Source{},
&digitalyama.Source{},
&submd.Source{},
}
@@ -174,9 +186,9 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup
}
}
// TODO: Consider refactoring this to avoid potential duplication issues
for _, source := range sources {
if source.NeedsKey() {
keyReq := source.KeyRequirement()
if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey {
if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" {
source.AddApiKeys([]string{apiKey})
}
pkg/passive/sources_test.go
+17 -5
View File
@@ -26,6 +26,7 @@ var (
"dnsdumpster",
"dnsdb",
"dnsrepo",
"domainsproject",
"driftnet",
"fofa",
"fullhunt",
@@ -33,6 +34,7 @@ var (
"hackertarget",
"intelx",
"netlas",
"onyphe",
"quake",
"pugrecon",
"rapiddns",
@@ -41,6 +43,7 @@ var (
"robtex",
"rsecloud",
"securitytrails",
"profundis",
"shodan",
"sitedossier",
"threatbook",
@@ -48,15 +51,18 @@ var (
"virustotal",
"waybackarchive",
"whoisxmlapi",
"windvane",
"zoomeyeapi",
"hunter",
"leakix",
"facebook",
// "threatminer",
// "reconcloud",
"reconeer",
"builtwith",
"hudsonrock",
"digitalyama",
"merklemap",
"thc",
"urlscan",
"submd",
}
@@ -73,28 +79,33 @@ var (
"crtsh",
"digitorus",
"dnsdumpster",
"domainsproject",
"dnsrepo",
"driftnet",
"fofa",
"fullhunt",
"hackertarget",
"intelx",
"onyphe",
"quake",
"redhuntlabs",
"robtex",
// "riddler", // failing due to cloudfront protection
"rsecloud",
"securitytrails",
"profundis",
"shodan",
"windvane",
"virustotal",
"whoisxmlapi",
"hunter",
"leakix",
"facebook",
// "threatminer",
// "reconcloud",
"reconeer",
"builtwith",
"digitalyama",
"thc",
"urlscan",
"submd",
}
@@ -110,7 +121,8 @@ var (
"securitytrails",
"virustotal",
"leakix",
"facebook",
"merklemap",
"urlscan",
// "reconcloud",
}
)
pkg/passive/sources_wo_auth_test.go
+4
View File
@@ -33,6 +33,10 @@ func TestSourcesWithoutKeys(t *testing.T) {
"digitorus", // failing with "Failed to retrieve certificate"
"dnsdumpster", // failing with "unexpected status code 403 received"
"anubis", // failing with "too many redirects"
"threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times"
"leakix", // now requires API key (returns 401)
"reconeer", // now requires API key (returns 401)
"sitedossier", // flaky - returns no results in CI
}
domain := "hackerone.com"
pkg/resolve/resolve.go
+13 -11
View File
@@ -25,18 +25,20 @@ type ResolutionPool struct {
// HostEntry defines a host with the source
type HostEntry struct {
Domain string
Host string
Source string
Domain string
Host string
Source string
WildcardCertificate bool
}
// Result contains the result for a host resolution
type Result struct {
Type ResultType
Host string
IP string
Error error
Source string
Type ResultType
Host string
IP string
Error error
Source string
WildcardCertificate bool
}
// ResultType is the type of result found
@@ -92,13 +94,13 @@ func (r *ResolutionPool) InitWildcards(domain string) error {
func (r *ResolutionPool) resolveWorker() {
for task := range r.Tasks {
if !r.removeWildcard {
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source}
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate}
continue
}
hosts, err := r.DNSClient.Lookup(task.Host)
if err != nil {
r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err}
r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate}
continue
}
@@ -116,7 +118,7 @@ func (r *ResolutionPool) resolveWorker() {
}
if !skip {
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate}
}
}
r.wg.Done()
pkg/runner/banners.go
+1 -1
View File
@@ -17,7 +17,7 @@ const banner = `
const ToolName = `subfinder`
// Version is the current version of subfinder
const version = `v2.8.0`
const version = `v2.13.0`
// showBanner is used to show the banner to the user
func showBanner() {
pkg/runner/config.go
+4 -2
View File
@@ -8,6 +8,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/subfinder/v2/pkg/passive"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
fileutil "github.com/projectdiscovery/utils/file"
)
@@ -25,7 +26,8 @@ func createProviderConfigYAML(configFilePath string) error {
sourcesRequiringApiKeysMap := make(map[string][]string)
for _, source := range passive.AllSources {
if source.NeedsKey() {
keyReq := source.KeyRequirement()
if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey {
sourceName := strings.ToLower(source.Name())
sourcesRequiringApiKeysMap[sourceName] = []string{}
}
@@ -46,7 +48,7 @@ func UnmarshalFrom(file string) error {
for _, source := range passive.AllSources {
sourceName := strings.ToLower(source.Name())
apiKeys := sourceApiKeysMap[sourceName]
if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 {
if len(apiKeys) > 0 {
gologger.Debug().Msgf("API key(s) found for %s.", sourceName)
source.AddApiKeys(apiKeys)
}
pkg/runner/enumerate.go
+28 -12
View File
@@ -67,6 +67,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error)
case subscraping.Subdomain:
subdomain := replacer.Replace(result.Value)
// check if this subdomain is actually a wildcard subdomain
// that may have furthur subdomains associated with it
isWildcard := strings.Contains(result.Value, "*."+subdomain)
// Validate the subdomain found and remove wildcards from
if !strings.HasSuffix(subdomain, "."+domain) {
@@ -90,10 +93,20 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
// send the subdomain for resolution.
if _, ok := uniqueMap[subdomain]; ok {
skippedCounts[result.Source]++
// even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard
// then we should mark it as wildcard
if !uniqueMap[subdomain].WildcardCertificate && isWildcard {
val := uniqueMap[subdomain]
val.WildcardCertificate = true
uniqueMap[subdomain] = val
}
continue
}
hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source}
hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard}
if r.options.ResultCallback != nil && !r.options.RemoveWildcard {
r.options.ResultCallback(&hostEntry)
}
uniqueMap[subdomain] = hostEntry
// If the user asked to remove wildcard then send on the resolve
@@ -109,6 +122,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
if r.options.RemoveWildcard {
close(resolutionPool.Tasks)
}
wg.Done()
}()
@@ -125,9 +139,22 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
// Add the found subdomain to a map.
if _, ok := foundResults[result.Host]; !ok {
foundResults[result.Host] = result
if r.options.ResultCallback != nil {
r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate})
}
}
}
}
// Merge wildcard certificate information from uniqueMap into foundResults
// This handles cases where a later source marked a subdomain as wildcard
// after it was already sent to the resolution pool
for host, result := range foundResults {
if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate {
result.WildcardCertificate = true
foundResults[host] = result
}
}
}
wg.Wait()
outputWriter := NewOutputWriter(r.options.JSON)
@@ -162,17 +189,6 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
numberOfSubDomains = len(uniqueMap)
}
if r.options.ResultCallback != nil {
if r.options.RemoveWildcard {
for host, result := range foundResults {
r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source})
}
} else {
for _, v := range uniqueMap {
r.options.ResultCallback(&v)
}
}
}
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration)
if r.options.Statistics {
pkg/runner/options.go
+14 -7
View File
@@ -15,6 +15,8 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/subfinder/v2/pkg/passive"
"github.com/projectdiscovery/subfinder/v2/pkg/resolve"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
envutil "github.com/projectdiscovery/utils/env"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
logutil "github.com/projectdiscovery/utils/log"
@@ -23,8 +25,8 @@ import (
var (
configDir = folderutil.AppConfigDirOrDefault(".", "subfinder")
defaultConfigLocation = filepath.Join(configDir, "config.yaml")
defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml")
defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml"))
defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml"))
)
// Options contains the configuration options for tuning
@@ -222,16 +224,20 @@ func (options *Options) loadProvidersFrom(location string) {
func listSources(options *Options) {
gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources))
gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n")
gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n")
gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n")
gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig)
for _, source := range passive.AllSources {
message := "%s\n"
sourceName := source.Name()
if source.NeedsKey() {
message = "%s *\n"
switch source.KeyRequirement() {
case subscraping.RequiredKey:
gologger.Silent().Msgf("%s *\n", sourceName)
case subscraping.OptionalKey:
gologger.Silent().Msgf("%s ~\n", sourceName)
default:
gologger.Silent().Msgf("%s\n", sourceName)
}
gologger.Silent().Msgf(message, sourceName)
}
}
@@ -259,4 +265,5 @@ var defaultRateLimits = []string{
// "gitlab=2/s",
"github=83/m",
"hudsonrock=5/s",
"urlscan=1/s",
}
pkg/runner/outputter.go
+20 -15
View File
@@ -19,22 +19,25 @@ type OutputWriter struct {
}
type jsonSourceResult struct {
Host string `json:"host"`
Input string `json:"input"`
Source string `json:"source"`
Host string `json:"host"`
Input string `json:"input"`
Source string `json:"source"`
WildcardCertificate bool `json:"wildcard_certificate,omitempty"`
}
type jsonSourceIPResult struct {
Host string `json:"host"`
IP string `json:"ip"`
Input string `json:"input"`
Source string `json:"source"`
Host string `json:"host"`
IP string `json:"ip"`
Input string `json:"input"`
Source string `json:"source"`
WildcardCertificate bool `json:"wildcard_certificate,omitempty"`
}
type jsonSourcesResult struct {
Host string `json:"host"`
Input string `json:"input"`
Sources []string `json:"sources"`
Host string `json:"host"`
Input string `json:"input"`
Sources []string `json:"sources"`
WildcardCertificate bool `json:"wildcard_certificate,omitempty"`
}
// NewOutputWriter creates a new OutputWriter
@@ -117,7 +120,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io.
data.IP = result.IP
data.Input = input
data.Source = result.Source
data.WildcardCertificate = result.WildcardCertificate
err := encoder.Encode(&data)
if err != nil {
return err
@@ -130,7 +133,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io.
func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error {
hosts := make(map[string]resolve.HostEntry)
for host, result := range results {
hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source}
hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate}
}
return o.WriteHost(input, hosts, writer)
@@ -175,6 +178,7 @@ func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io
data.Host = result.Host
data.Input = input
data.Source = result.Source
data.WildcardCertificate = result.WildcardCertificate
err := encoder.Encode(data)
if err != nil {
return err
@@ -223,11 +227,12 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr
for host, sources := range sourceMap {
sb.WriteString(host)
sb.WriteString(",[")
sourcesString := ""
var sourcesString strings.Builder
for source := range sources {
sourcesString += source + ","
sourcesString.WriteString(source)
sourcesString.WriteRune(',')
}
sb.WriteString(strings.Trim(sourcesString, ", "))
sb.WriteString(strings.TrimSuffix(sourcesString.String(), ","))
sb.WriteString("]\n")
_, err := bufwriter.WriteString(sb.String())
pkg/runner/runner.go
+7
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"strconv"
"strings"
"time"
"github.com/projectdiscovery/gologger"
contextutil "github.com/projectdiscovery/utils/context"
@@ -61,11 +62,17 @@ func NewRunner(options *Options) (*Runner, error) {
Custom: mapsutil.SyncLockMap[string, uint]{
Map: make(map[string]uint),
},
CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{
Map: make(map[string]time.Duration),
},
}
for source, sourceRateLimit := range options.RateLimits.AsMap() {
if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint {
_ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount)
if sourceRateLimit.Duration > 0 {
_ = runner.rateLimit.CustomDuration.Set(source, sourceRateLimit.Duration)
}
}
}
pkg/runner/stats.go
+2 -2
View File
@@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) {
if sourceStats.Skipped {
skipped = append(skipped, fmt.Sprintf(" %s", source))
} else {
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors))
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors))
}
}
if len(lines) > 0 {
gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56))
gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68))
gologger.Print().Msg(strings.Join(lines, "\n"))
gologger.Print().Msgf("\n")
}
pkg/subscraping/agent.go
+1 -1
View File
@@ -46,7 +46,7 @@ func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLi
Timeout: time.Duration(timeout) * time.Second,
}
session := &Session{Client: client}
session := &Session{Client: client, Timeout: timeout}
// Initiate rate limit instance
session.MultiRateLimiter = multiRateLimiter
pkg/subscraping/sources/alienvault/alienvault.go
+29 -7
View File
@@ -23,6 +23,9 @@ type Source struct {
timeTaken time.Duration
results int
errors int
requests int
apiKeys []string
skipped bool
}
// Run function returns all subdomains found with the service
@@ -30,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -37,7 +41,15 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain))
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "",
map[string]string{"Authorization": "Bearer " + randomApiKey})
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
@@ -64,8 +76,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, record := range response.PassiveDNS {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}:
s.results++
}
}
}()
@@ -85,18 +101,24 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) NeedsKey() bool {
return false
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
// no key needed
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
pkg/subscraping/sources/anubis/anubis.go
+15 -3
View File
@@ -17,6 +17,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -56,8 +59,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
for _, record := range subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}:
s.results++
}
}
}()
@@ -78,8 +85,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -90,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
pkg/subscraping/sources/bevigil/bevigil.go
+15 -3
View File
@@ -21,6 +21,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
s.requests++
resp, err := session.Get(ctx, getUrl, "", map[string]string{
"X-Access-Token": randomApiKey, "User-Agent": "subfinder",
})
@@ -70,8 +73,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}()
@@ -90,8 +97,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -102,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/bufferover/bufferover.go
+15 -3
View File
@@ -27,6 +27,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) {
s.requests++
resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey})
if err != nil && resp == nil {
@@ -96,8 +99,12 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s
for _, subdomain := range subdomains {
for _, value := range session.Extractor.Extract(subdomain) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}:
s.results++
}
}
}
}
@@ -115,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -127,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/builtwith/builtwith.go
+15 -3
View File
@@ -35,6 +35,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -74,8 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
for _, result := range data.Results {
for _, path := range result.Result.Paths {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}:
s.results++
}
}
}
}()
@@ -96,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -108,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/c99/c99.go
+15 -3
View File
@@ -18,6 +18,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain)
s.requests++
resp, err := session.SimpleGet(ctx, searchURL)
if err != nil {
session.DiscardHTTPResponse(resp)
@@ -81,8 +84,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for _, data := range response.Subdomains {
if !strings.HasPrefix(data.Subdomain, ".") {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}:
s.results++
}
}
}
}()
@@ -103,8 +110,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -115,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/censys/censys.go
+106 -54
View File
@@ -2,55 +2,71 @@
package censys
import (
"bytes"
"context"
"strconv"
"net/http"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
urlutil "github.com/projectdiscovery/utils/url"
)
const (
// maxCensysPages is the maximum number of pages to fetch from the API
maxCensysPages = 10
maxPerPage = 100
// maxPerPage is the maximum number of results per page
maxPerPage = 100
// baseURL is the Censys Platform API base URL
baseURL = "https://api.platform.censys.io"
// searchEndpoint is the global data search query endpoint
searchEndpoint = "/v3/global/search/query"
// queryPrefix is the Censys query language prefix for certificate name search
queryPrefix = "cert.names: "
// authHeaderPrefix is the Bearer token prefix for Authorization header
authHeaderPrefix = "Bearer "
// contentTypeJSON is the Content-Type header value for JSON
contentTypeJSON = "application/json"
// orgIDHeader is the header name for organization ID
orgIDHeader = "X-Organization-ID"
)
// apiKey holds the Personal Access Token and optional Organization ID
type apiKey struct {
pat string
orgID string
}
// Platform API request body
type searchRequest struct {
Query string `json:"query"`
Fields []string `json:"fields,omitempty"`
PageSize int `json:"page_size,omitempty"`
Cursor string `json:"cursor,omitempty"`
}
// Platform API response structures
type response struct {
Code int `json:"code"`
Status string `json:"status"`
Result result `json:"result"`
}
type result struct {
Query string `json:"query"`
Total float64 `json:"total"`
DurationMS int `json:"duration_ms"`
Hits []hit `json:"hits"`
Links links `json:"links"`
Hits []hit `json:"hits"`
TotalHits int64 `json:"total_hits"`
NextPageToken string `json:"next_page_token"`
}
type hit struct {
Parsed parsed `json:"parsed"`
Names []string `json:"names"`
FingerprintSha256 string `json:"fingerprint_sha256"`
CertificateV1 certificateV1 `json:"certificate_v1"`
}
type parsed struct {
ValidityPeriod validityPeriod `json:"validity_period"`
SubjectDN string `json:"subject_dn"`
IssuerDN string `json:"issuer_dn"`
type certificateV1 struct {
Resource resource `json:"resource"`
}
type validityPeriod struct {
NotAfter string `json:"not_after"`
NotBefore string `json:"not_before"`
}
type links struct {
Next string `json:"next"`
Prev string `json:"prev"`
type resource struct {
Names []string `json:"names"`
}
// Source is the passive scraping agent
@@ -59,19 +75,16 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
type apiKey struct {
token string
secret string
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -79,37 +92,61 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
// PickRandom selects a random API key from configured keys.
// This enables load balancing when users configure multiple PATs
// (e.g., CENSYS_API_KEY=pat1:org1,pat2:org2) to distribute requests
// and avoid hitting rate limits on a single key.
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey.token == "" || randomApiKey.secret == "" {
if randomApiKey.pat == "" {
s.skipped = true
return
}
certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search"
apiURL := baseURL + searchEndpoint
cursor := ""
currentPage := 1
for {
certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint)
select {
case <-ctx.Done():
return
default:
}
reqBody := searchRequest{
Query: queryPrefix + domain,
Fields: []string{"cert.names"},
PageSize: maxPerPage,
}
if cursor != "" {
reqBody.Cursor = cursor
}
bodyBytes, err := jsoniter.Marshal(reqBody)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
certSearchEndpointUrl.Params.Add("q", domain)
certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage))
if cursor != "" {
certSearchEndpointUrl.Params.Add("cursor", cursor)
headers := map[string]string{
"Content-Type": contentTypeJSON,
"Authorization": authHeaderPrefix + randomApiKey.pat,
}
// Add Organization ID header if provided
if randomApiKey.orgID != "" {
headers[orgIDHeader] = randomApiKey.orgID
}
s.requests++
resp, err := session.HTTPRequest(
ctx,
"GET",
certSearchEndpointUrl.String(),
http.MethodPost,
apiURL,
"",
nil,
nil,
subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret},
headers,
bytes.NewReader(bodyBytes),
subscraping.BasicAuth{},
)
if err != nil {
@@ -121,24 +158,25 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
var censysResponse response
err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse)
_ = resp.Body.Close()
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)
for _, hit := range censysResponse.Result.Hits {
for _, name := range hit.Names {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}
s.results++
for _, name := range hit.CertificateV1.Resource.Names {
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}:
s.results++
}
}
}
// Exit the censys enumeration if last page is reached
cursor = censysResponse.Result.Links.Next
cursor = censysResponse.Result.NextPageToken
if cursor == "" || currentPage >= maxCensysPages {
break
}
@@ -162,20 +200,34 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) NeedsKey() bool {
return true
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
// AddApiKeys parses and adds API keys.
// Format: "PAT:ORG_ID" where ORG_ID is required for paid accounts.
// Example: "censys_xxx_token:12345678-91011-1213"
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
return apiKey{k, v}
s.apiKeys = subscraping.CreateApiKeys(keys, func(pat, orgID string) apiKey {
return apiKey{pat: pat, orgID: orgID}
})
// Also support single PAT without org ID for free users
for _, key := range keys {
if !strings.Contains(key, ":") && key != "" {
s.apiKeys = append(s.apiKeys, apiKey{pat: key, orgID: ""})
}
}
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/censys/censys_test.go
+137
View File
@@ -0,0 +1,137 @@
package censys
import (
"context"
"math"
"net/http"
"testing"
"time"
"github.com/projectdiscovery/ratelimit"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// createTestMultiRateLimiter creates a MultiLimiter for testing
func createTestMultiRateLimiter(ctx context.Context) *ratelimit.MultiLimiter {
mrl, _ := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{
Key: "censys",
IsUnlimited: false,
MaxCount: math.MaxInt32,
Duration: time.Millisecond,
})
return mrl
}
func TestCensysSource_NoApiKey(t *testing.T) {
source := &Source{}
// Don't add any API keys
ctx := context.Background()
multiRateLimiter := createTestMultiRateLimiter(ctx)
session := &subscraping.Session{
Client: http.DefaultClient,
MultiRateLimiter: multiRateLimiter,
}
ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "censys")
results := source.Run(ctxWithValue, "example.com", session)
// Collect all results
var resultCount int
for range results {
resultCount++
}
// Should be skipped when no API key
stats := source.Statistics()
assert.True(t, stats.Skipped, "expected source to be skipped without API key")
assert.Equal(t, 0, resultCount, "expected no results when skipped")
}
func TestCensysSource_ContextCancellation(t *testing.T) {
source := &Source{}
// Add a key with PAT:ORG_ID format
source.AddApiKeys([]string{"test_pat:test_org_id"})
ctx := context.Background()
multiRateLimiter := createTestMultiRateLimiter(ctx)
session := &subscraping.Session{
Client: http.DefaultClient,
MultiRateLimiter: multiRateLimiter,
}
// Create a context that will be cancelled
ctxCancellable, cancel := context.WithCancel(ctx)
ctxWithValue := context.WithValue(ctxCancellable, subscraping.CtxSourceArg, "censys")
results := source.Run(ctxWithValue, "example.com", session)
// Cancel immediately
cancel()
// Should exit quickly without blocking
done := make(chan struct{})
go func() {
for range results {
// drain
}
close(done)
}()
select {
case <-done:
// Good - completed quickly
case <-time.After(2 * time.Second):
t.Fatal("context cancellation did not stop the source in time")
}
}
func TestCensysSource_Metadata(t *testing.T) {
source := &Source{}
assert.Equal(t, "censys", source.Name())
assert.True(t, source.IsDefault())
assert.False(t, source.HasRecursiveSupport())
assert.True(t, source.NeedsKey())
}
func TestCensysSource_AddApiKeys(t *testing.T) {
t.Run("PAT with OrgID", func(t *testing.T) {
source := &Source{}
keys := []string{"pat_token_1:org_id_1", "pat_token_2:org_id_2"}
source.AddApiKeys(keys)
require.Len(t, source.apiKeys, 2)
assert.Equal(t, "pat_token_1", source.apiKeys[0].pat)
assert.Equal(t, "org_id_1", source.apiKeys[0].orgID)
assert.Equal(t, "pat_token_2", source.apiKeys[1].pat)
assert.Equal(t, "org_id_2", source.apiKeys[1].orgID)
})
t.Run("PAT without OrgID (free user)", func(t *testing.T) {
source := &Source{}
keys := []string{"pat_token_only"}
source.AddApiKeys(keys)
require.Len(t, source.apiKeys, 1)
assert.Equal(t, "pat_token_only", source.apiKeys[0].pat)
assert.Equal(t, "", source.apiKeys[0].orgID)
})
}
func TestCensysSource_Statistics(t *testing.T) {
source := &Source{
errors: 2,
results: 10,
timeTaken: 5 * time.Second,
skipped: false,
}
stats := source.Statistics()
assert.Equal(t, 2, stats.Errors)
assert.Equal(t, 10, stats.Results)
assert.Equal(t, 5*time.Second, stats.TimeTaken)
assert.False(t, stats.Skipped)
}
pkg/subscraping/sources/certspotter/certspotter.go
+27 -6
View File
@@ -22,6 +22,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
headers := map[string]string{"Authorization": "Bearer " + randomApiKey}
cookies := ""
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -66,20 +69,29 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for _, cert := range response {
for _, subdomain := range cert.DNSNames {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
// if the number of responses is zero, close the channel and return.
if len(response) == 0 {
return
}
id := response[len(response)-1].ID
for {
select {
case <-ctx.Done():
return
default:
}
reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
s.requests++
resp, err := session.Get(ctx, reqURL, cookies, headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -103,8 +115,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for _, cert := range response {
for _, subdomain := range cert.DNSNames {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
@@ -128,8 +144,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -140,6 +160,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/chaos/chaos.go
+15 -2
View File
@@ -16,14 +16,16 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result {
func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -38,9 +40,15 @@ func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <
}
chaosClient := chaos.New(randomApiKey)
s.requests++
for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{
Domain: domain,
}) {
select {
case <-ctx.Done():
return
default:
}
if result.Error != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error}
s.errors++
@@ -69,8 +77,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -81,6 +93,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/chinaz/chinaz.go
+20 -3
View File
@@ -17,6 +17,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -55,9 +58,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if SubdomainList.ToBool() {
_data := []byte(SubdomainList.ToString())
for i := 0; i < SubdomainList.Size(); i++ {
select {
case <-ctx.Done():
return
default:
}
subdomain := jsoniter.Get(_data, i, "DataUrl").ToString()
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
} else {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -82,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -94,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/commoncrawl/commoncrawl.go
+23 -4
View File
@@ -32,6 +32,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, indexURL)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -105,8 +108,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -117,6 +124,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
@@ -128,6 +136,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se
return false
default:
var headers = map[string]string{"Host": "index.commoncrawl.org"}
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -138,6 +147,12 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return false
default:
}
line := scanner.Text()
if line == "" {
continue
@@ -145,13 +160,17 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se
line, _ = url.QueryUnescape(line)
for _, subdomain := range session.Extractor.Extract(line) {
if subdomain != "" {
// fix for triple encoded URL
subdomain = strings.ToLower(subdomain)
subdomain = strings.TrimPrefix(subdomain, "25")
subdomain = strings.TrimPrefix(subdomain, "2f")
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return false
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}
pkg/subscraping/sources/crtsh/crtsh.go
+35 -7
View File
@@ -29,6 +29,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -54,7 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int {
db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes")
// connect_timeout: limits connection establishment time (in seconds)
// statement_timeout: limits query execution time (in milliseconds)
connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000)
db, err := sql.Open("postgres", connStr)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
@@ -114,8 +119,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio
var count int
var data string
// Parse all the rows getting subdomains
for rows.Next() {
select {
case <-ctx.Done():
return count
default:
}
err := rows.Scan(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -127,8 +136,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio
for subdomain := range strings.SplitSeq(data, "\n") {
for _, value := range session.Extractor.Extract(subdomain) {
if value != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
s.results++
select {
case <-ctx.Done():
return count
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}:
s.results++
}
}
}
}
@@ -137,6 +150,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio
}
func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -157,11 +171,20 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi
session.DiscardHTTPResponse(resp)
for _, subdomain := range subdomains {
select {
case <-ctx.Done():
return true
default:
}
for sub := range strings.SplitSeq(subdomain.NameValue, "\n") {
for _, value := range session.Extractor.Extract(sub) {
if value != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
s.results++
select {
case <-ctx.Done():
return true
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}:
s.results++
}
}
}
}
@@ -183,8 +206,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -195,6 +222,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
pkg/subscraping/sources/digitalyama/digitalyama.go
+15 -3
View File
@@ -16,6 +16,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -34,6 +35,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -48,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain)
s.requests++
resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -94,8 +97,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range response.Subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}()
@@ -115,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -127,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/digitorus/digitorus.go
+19 -4
View File
@@ -18,6 +18,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -32,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain))
// the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774
if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound {
@@ -50,16 +53,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
return
default:
}
line := scanner.Text()
if line == "" {
continue
}
subdomains := session.Extractor.Extract(line)
for _, subdomain := range subdomains {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."),
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, ".")}:
s.results++
}
s.results++
}
}
}()
@@ -80,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -92,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
pkg/subscraping/sources/dnsdb/dnsdb.go
+27 -10
View File
@@ -43,6 +43,7 @@ type Source struct {
timeTaken time.Duration
errors int
results uint64
requests int
skipped bool
}
@@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -70,6 +72,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
"Accept": "application/x-ndjson",
}
s.requests++
offsetMax, err := getMaxOffset(ctx, session, headers)
if err != nil {
results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err}
@@ -85,8 +88,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
queryParams.Add("swclient", "subfinder")
for {
select {
case <-ctx.Done():
return
default:
}
url := urlTemplate + queryParams.Encode()
s.requests++
resp, err := session.Get(ctx, url, "", headers)
if err != nil {
results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err}
@@ -98,6 +107,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
var respCond string
reader := bufio.NewReader(resp.Body)
for {
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
default:
}
n, err := reader.ReadBytes('\n')
if err == io.EOF {
break
@@ -117,21 +132,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
// Condition is a scalar enum of string values: {“begin”, “ongoing”, “succeeded”, “limited”, “failed”}.
// "begin" will be the initiating Condition, this can be safely ignored. The data of interest will be in
// objects with Condition "" or "ongoing". Conditions "succeeded", "limited", and "failed" are terminating conditions.
// See https://www.domaintools.com/resources/user-guides/farsight-streaming-api-framing-protocol-documentation/
// for more details
respCond = response.Condition
if respCond == "" || respCond == "ongoing" {
if response.Obj.Name != "" {
results <- subscraping.Result{
Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, "."),
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
case results <- subscraping.Result{Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, ".")}:
s.results++
}
s.results++
}
} else if respCond != "begin" {
// if the respCond is not "", "ongoing", or "begin", then it is a terminating condition, so break out of the loop
break
}
}
@@ -175,8 +187,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -187,6 +203,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: int(s.results),
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/dnsdumpster/dnsdumpster.go
+15 -3
View File
@@ -25,6 +25,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -65,8 +68,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, record := range append(response.A, response.Ns...) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}:
s.results++
}
}
}()
@@ -87,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -99,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/dnsrepo/dnsrepo.go
+14 -4
View File
@@ -17,6 +17,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
token := randomApiInfo[0]
apiKey := randomApiInfo[1]
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -74,10 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
for _, sub := range result {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."),
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}:
s.results++
}
s.results++
}
}()
@@ -98,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -110,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/domainsproject/domainsproject.go
+145
View File
@@ -0,0 +1,145 @@
// Package domainsproject logic
package domainsproject
import (
"context"
"fmt"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct {
apiKeys []apiKey
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
type apiKey struct {
username string
password string
}
type domainsProjectResponse struct {
Domains []string `json:"domains"`
Error string `json:"error"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey.username == "" || randomApiKey.password == "" {
s.skipped = true
return
}
searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain)
s.requests++
resp, err := session.HTTPRequest(
ctx,
"GET",
searchURL,
"",
nil,
nil,
subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password},
)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
}
}()
var response domainsProjectResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
if response.Error != "" {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error),
}
s.errors++
return
}
for _, subdomain := range response.Domains {
if !strings.HasPrefix(subdomain, ".") {
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "domainsproject"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
return apiKey{k, v}
})
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
pkg/subscraping/sources/driftnet/driftnet.go
+22 -6
View File
@@ -28,6 +28,7 @@ type Source struct {
timeTaken time.Duration
errors atomic.Int32
results atomic.Int32
requests atomic.Int32
skipped bool
}
@@ -65,6 +66,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors.Store(0)
s.results.Store(0)
s.requests.Store(0)
// Waitgroup for subsources
var wg sync.WaitGroup
@@ -104,9 +106,14 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
// KeyRequirement indicates that we need an API key
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
// NeedsKey indicates that we need an API key
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
// AddApiKeys provides us with the API key(s)
@@ -119,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: int(s.errors.Load()),
Results: int(s.results.Load()),
Requests: int(s.requests.Load()),
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
@@ -139,6 +147,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc
// Request
requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit)
s.requests.Add(1)
resp, err := session.Get(ctx, requestURL, "", headers)
if err != nil {
// HTTP 204 is not an error from the Driftnet API
@@ -176,17 +185,24 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc
}
for subdomain := range summary.Summary.Values {
// We can get certificate results which aren't actually subdomains of the target domain. Skip them.
select {
case <-ctx.Done():
wg.Done()
return
default:
}
if !strings.HasSuffix(subdomain, "."+domain) {
continue
}
// Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints)
if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain,
select {
case <-ctx.Done():
wg.Done()
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results.Add(1)
}
s.results.Add(1)
}
}
pkg/subscraping/sources/facebook/ctlogs.go
-200
View File
@@ -1,200 +0,0 @@
package facebook
import (
"context"
"encoding/json"
"fmt"
"io"
"time"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
errorutil "github.com/projectdiscovery/utils/errors"
"github.com/projectdiscovery/utils/generic"
urlutil "github.com/projectdiscovery/utils/url"
)
// source: https://developers.facebook.com/tools/ct
// api-docs: https://developers.facebook.com/docs/certificate-transparency-api
// ratelimit: ~20,000 req/hour per appID https://developers.facebook.com/docs/graph-api/overview/rate-limiting/
var (
domainsPerPage = "1000"
authUrl = "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials"
domainsUrl = "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=%s&limit=" + domainsPerPage
)
type apiKey struct {
AppID string
Secret string
AccessToken string // obtained by calling
// https://graph.facebook.com/oauth/access_token?client_id=APP_ID&client_secret=APP_SECRET&grant_type=client_credentials
Error error // error while fetching access token
}
// FetchAccessToken fetches the access token for the api key
// using app id and secret
func (k *apiKey) FetchAccessToken() {
if generic.EqualsAny("", k.AppID, k.Secret) {
k.Error = fmt.Errorf("invalid app id or secret")
return
}
resp, err := retryablehttp.Get(fmt.Sprintf(authUrl, k.AppID, k.Secret))
if err != nil {
k.Error = err
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
gologger.Error().Msgf("error closing response body: %s", err)
}
}()
bin, err := io.ReadAll(resp.Body)
if err != nil {
k.Error = err
return
}
auth := &authResponse{}
if err := json.Unmarshal(bin, auth); err != nil {
k.Error = err
return
}
if auth.AccessToken == "" {
k.Error = fmt.Errorf("invalid response from facebook got %v", string(bin))
return
}
k.AccessToken = auth.AccessToken
}
// IsValid returns true if the api key is valid
func (k *apiKey) IsValid() bool {
return k.AccessToken != ""
}
// Source is the passive scraping agent
type Source struct {
apiKeys []apiKey
timeTaken time.Duration
errors int
results int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
if len(s.apiKeys) == 0 {
s.skipped = true
close(results)
return results
}
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
key := subscraping.PickRandom(s.apiKeys, s.Name())
domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain)
for {
// unfortunately, this cannot be parllelized since pagination is cursor based
resp, err := session.Get(ctx, domainsURL, "", nil)
if err != nil {
s.errors++
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
bin, err := io.ReadAll(resp.Body)
if err != nil {
s.errors++
gologger.Verbose().Msgf("failed to read response body: %s\n", err)
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
session.DiscardHTTPResponse(resp)
response := &response{}
if err := json.Unmarshal(bin, response); err != nil {
s.errors++
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))}
return
}
for _, v := range response.Data {
for _, domain := range v.Domains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}
s.results++
}
}
if response.Paging.Next == "" {
break
}
// cursor includes api key so no need to update it
domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage)
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "facebook"
}
// IsDefault returns true if the source should be queried by default
func (s *Source) IsDefault() bool {
return true
}
// accepts subdomains (e.g. subdomain.domain.tld)
// but also returns all SANs for a certificate which may not match the domain
func (s *Source) HasRecursiveSupport() bool {
return true
}
// NeedsKey returns true if the source requires an API key
func (s *Source) NeedsKey() bool {
return true
}
// AddApiKeys adds api keys to the source
func (s *Source) AddApiKeys(keys []string) {
allapikeys := subscraping.CreateApiKeys(keys, func(k, v string) apiKey {
apiKey := apiKey{AppID: k, Secret: v}
apiKey.FetchAccessToken()
if apiKey.Error != nil {
gologger.Warning().Msgf("Could not fetch access token for %s: %s\n", k, apiKey.Error)
}
return apiKey
})
// filter out invalid keys
for _, key := range allapikeys {
if key.IsValid() {
s.apiKeys = append(s.apiKeys, key)
}
}
}
// Statistics returns the statistics for the source
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
func updateParamInURL(url, param, value string) string {
urlx, err := urlutil.Parse(url)
if err != nil {
return url
}
urlx.Params.Set(param, value)
return urlx.String()
}
pkg/subscraping/sources/facebook/ctlogs_test.go
-71
View File
@@ -1,71 +0,0 @@
package facebook
import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"testing"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/generic"
)
var (
fb_API_ID = "$FB_APP_ID"
fb_API_SECRET = "$FB_APP_SECRET"
)
func TestFacebookSource(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
updateWithEnv(&fb_API_ID)
updateWithEnv(&fb_API_SECRET)
if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) {
t.SkipNow()
}
k := apiKey{
AppID: fb_API_ID,
Secret: fb_API_SECRET,
}
k.FetchAccessToken()
if k.Error != nil {
t.Fatal(k.Error)
}
fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken)
resp, err := retryablehttp.Get(fetchURL)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
gologger.Error().Msgf("error closing response body: %s", err)
}
}()
bin, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
response := &response{}
if err := json.Unmarshal(bin, response); err != nil {
t.Fatal(err)
}
if len(response.Data) == 0 {
t.Fatal("no data found")
}
}
func updateWithEnv(key *string) {
if key == nil {
return
}
value := *key
if strings.HasPrefix(value, "$") {
*key = os.Getenv(value[1:])
}
}
pkg/subscraping/sources/facebook/types.go
-36
View File
@@ -1,36 +0,0 @@
package facebook
type authResponse struct {
AccessToken string `json:"access_token"`
}
/*
{
"data": [
{
"domains": [
"docs.hackerone.com"
],
"id": "10056051421102939"
},
...
],
"paging": {
"cursors": {
"before": "MTAwNTYwNTE0MjExMDI5MzkZD",
"after": "Njc0OTczNTA5NTA1MzUxNwZDZD"
},
"next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD"
}
}
*/
// example response
type response struct {
Data []struct {
Domains []string `json:"domains"`
} `json:"data"`
Paging struct {
Next string `json:"next"`
} `json:"paging"`
}
pkg/subscraping/sources/fofa/fofa.go
+14 -1
View File
@@ -27,6 +27,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
// fofa api doc https://fofa.info/static_pages/api_help
qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -83,6 +86,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if response.Size > 0 {
for _, subdomain := range response.Results {
select {
case <-ctx.Done():
return
default:
}
if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") {
subdomain = subdomain[strings.Index(subdomain, "//")+2:]
}
@@ -112,8 +120,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -126,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/fullhunt/fullhunt.go
+15 -3
View File
@@ -23,6 +23,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -61,8 +64,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
session.DiscardHTTPResponse(resp)
for _, record := range response.Hosts {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}:
s.results++
}
}
}()
@@ -82,8 +89,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -94,6 +105,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/github/github.go
+61 -17
View File
@@ -12,6 +12,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
jsoniter "github.com/json-iterator/go"
@@ -41,16 +42,18 @@ type response struct {
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
errors atomic.Int32
results atomic.Int32
requests atomic.Int32
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.errors.Store(0)
s.results.Store(0)
s.requests.Store(0)
go func() {
defer func(startTime time.Time) {
@@ -86,11 +89,12 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
}
// Initial request to GitHub search
s.requests.Add(1)
resp, err := session.Get(ctx, searchURL, "", headers)
isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
if err != nil && !isForbidden {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
session.DiscardHTTPResponse(resp)
return
}
@@ -111,7 +115,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
session.DiscardHTTPResponse(resp)
return
}
@@ -121,7 +125,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
return
}
@@ -129,11 +133,16 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
linksHeader := linkheader.Parse(resp.Header.Get("Link"))
// Process the next link recursively
for _, link := range linksHeader {
select {
case <-ctx.Done():
return
default:
}
if link.Rel == "next" {
nextURL, err := url.QueryUnescape(link.URL)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
return
}
s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results)
@@ -147,11 +156,22 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r
errChan := make(chan error, len(items))
for _, responseItem := range items {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
wg.Add(1)
go func(responseItem item) {
defer wg.Done()
// find subdomains in code
select {
case <-ctx.Done():
return
default:
}
s.requests.Add(1)
resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL))
if err != nil {
if resp != nil && resp.StatusCode != http.StatusNotFound {
@@ -164,23 +184,42 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
default:
}
line := scanner.Text()
if line == "" {
continue
}
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}:
s.results.Add(1)
}
}
}
session.DiscardHTTPResponse(resp)
}
// find subdomains in text matches
for _, textMatch := range responseItem.TextMatches {
select {
case <-ctx.Done():
return
default:
}
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}:
s.results.Add(1)
}
}
}
}(responseItem)
@@ -231,8 +270,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -241,8 +284,9 @@ func (s *Source) AddApiKeys(keys []string) {
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Errors: int(s.errors.Load()),
Results: int(s.results.Load()),
Requests: int(s.requests.Load()),
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/gitlab/gitlab.go
+27 -14
View File
@@ -9,6 +9,7 @@ import (
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
jsoniter "github.com/json-iterator/go"
@@ -20,8 +21,9 @@ import (
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
errors atomic.Int32
results atomic.Int32
requests atomic.Int32
skipped bool
}
@@ -35,8 +37,9 @@ type item struct {
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.errors.Store(0)
s.results.Store(0)
s.requests.Store(0)
go func() {
defer func(startTime time.Time) {
@@ -66,10 +69,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
default:
}
s.requests.Add(1)
resp, err := session.Get(ctx, searchURL, "", headers)
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
session.DiscardHTTPResponse(resp)
return
}
@@ -80,7 +84,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
err = jsoniter.NewDecoder(resp.Body).Decode(&items)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
return
}
@@ -91,13 +95,14 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
go func(item item) {
// The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path
fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref)
s.requests.Add(1)
resp, err := session.Get(ctx, fileUrl, "", headers)
if err != nil {
if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) {
session.DiscardHTTPResponse(resp)
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
return
}
}
@@ -111,7 +116,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
}
for _, subdomain := range domainRegexp.FindAllString(line, -1) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
s.results.Add(1)
}
}
session.DiscardHTTPResponse(resp)
@@ -120,15 +125,18 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *
}(it)
}
// Links header, first, next, last...
linksHeader := linkheader.Parse(resp.Header.Get("Link"))
// Process the next link recursively
for _, link := range linksHeader {
select {
case <-ctx.Done():
return
default:
}
if link.Rel == "next" {
nextURL, err := url.QueryUnescape(link.URL)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
s.errors.Add(1)
return
}
@@ -157,8 +165,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -168,8 +180,9 @@ func (s *Source) AddApiKeys(keys []string) {
// Statistics returns the statistics for the source
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Errors: int(s.errors.Load()),
Results: int(s.results.Load()),
Requests: int(s.requests.Load()),
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/hackertarget/hackertarget.go
+35 -7
View File
@@ -12,9 +12,12 @@ import (
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
@@ -22,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -29,7 +33,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain))
htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain)
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey != "" {
htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey)
}
htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey)
s.requests++
resp, err := session.SimpleGet(ctx, htSearchUrl)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
@@ -46,14 +59,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
return
default:
}
line := scanner.Text()
if line == "" {
continue
}
match := session.Extractor.Extract(line)
for _, subdomain := range match {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}()
@@ -74,12 +96,16 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) NeedsKey() bool {
return false
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.OptionalKey
}
func (s *Source) AddApiKeys(_ []string) {
// no key needed
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
@@ -87,5 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/hudsonrock/hudsonrock.go
+15 -3
View File
@@ -26,6 +26,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -60,8 +63,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) {
for _, subdomain := range session.Extractor.Extract(record.URL) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
@@ -83,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -95,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
pkg/subscraping/sources/hunter/hunter.go
-131
View File
@@ -1,131 +0,0 @@
package hunter
import (
"context"
"encoding/base64"
"fmt"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type hunterResp struct {
Code int `json:"code"`
Data hunterData `json:"data"`
Message string `json:"message"`
}
type infoArr struct {
URL string `json:"url"`
IP string `json:"ip"`
Port int `json:"port"`
Domain string `json:"domain"`
Protocol string `json:"protocol"`
}
type hunterData struct {
InfoArr []infoArr `json:"arr"`
Total int `json:"total"`
}
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
var pages = 1
for currentPage := 1; currentPage <= pages; currentPage++ {
// hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2
qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var response hunterResp
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)
if response.Code == 401 || response.Code == 400 {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
}
s.errors++
return
}
if response.Data.Total > 0 {
for _, hunterInfo := range response.Data.InfoArr {
subdomain := hunterInfo.Domain
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
}
pages = int(response.Data.Total/1000) + 1
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "hunter"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) NeedsKey() bool {
return true
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
pkg/subscraping/sources/intelx/intelx.go
+20 -4
View File
@@ -43,6 +43,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -56,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -85,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -107,6 +110,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID)
status := 0
for status == 0 || status == 3 {
select {
case <-ctx.Done():
return
default:
}
s.requests++
resp, err = session.Get(ctx, resultsURL, "", nil)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -134,10 +143,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
status = response.Status
for _, hostname := range response.Selectors {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue,
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}:
s.results++
}
s.results++
}
}
}()
@@ -158,8 +169,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -172,6 +187,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/leakix/leakix.go
+14 -4
View File
@@ -16,6 +16,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
headers["api-key"] = randomApiKey
}
// Request
s.requests++
resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -64,10 +67,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
for _, result := range subdomains {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain,
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}:
s.results++
}
s.results++
}
}()
return results
@@ -86,8 +91,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -98,6 +107,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/merklemap/merklemap.go
+174
View File
@@ -0,0 +1,174 @@
// Package merklemap logic
package merklemap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net/url"
"strconv"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
// Pick an API key, skip if no key is found
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
// Default headers
headers := map[string]string{
"accept": "application/json",
// Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api
"User-Agent": "subfinder",
"Authorization": "Bearer " + randomApiKey,
}
// Fetch all pages with pagination
// https://www.merklemap.com/documentation/search
s.fetchAllPages(ctx, domain, headers, session, results)
}()
return results
}
// fetchAllPages fetches all pages of results using pagination
func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) {
baseURL := "https://api.merklemap.com/v1/search?query=" + url.QueryEscape("*."+domain)
totalCount := math.MaxInt
processedResults := 0
// Iterate through all pages
for page := 0; processedResults < totalCount; page++ {
pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
if page == 0 {
totalCount = pageResp.Count
}
// Stop if this page returned no results
if len(pageResp.Results) == 0 {
break
}
for _, result := range pageResp.Results {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: result.Hostname,
}
s.results++
processedResults++
}
}
}
// fetchPage fetches a single page of results
func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) {
url := baseURL + "&page=" + strconv.Itoa(page)
s.requests++
resp, err := session.Get(ctx, url, "", headers)
if err != nil {
return nil, err
}
defer session.DiscardHTTPResponse(resp)
if resp.StatusCode != 200 {
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, err)
}
return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, string(respBody))
}
var pageResponse response
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
decoder := json.NewDecoder(bytes.NewReader(respBody))
if err := decoder.Decode(&pageResponse); err != nil {
return nil, err
}
return &pageResponse, nil
}
// Name returns the name of the source
func (s *Source) Name() string {
return "merklemap"
}
func (s *Source) IsDefault() bool {
return false
}
// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains
func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
type response struct {
Count int `json:"count"`
Results []struct {
Hostname string `json:"hostname"`
SubjectCommonName string `json:"subject_common_name"`
FirstSeen string `json:"first_seen"`
} `json:"results"`
}
pkg/subscraping/sources/netlas/netlas.go
+25 -14
View File
@@ -40,6 +40,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -47,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -63,7 +65,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
// Pick an API key
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{
s.requests++
resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{
"accept": "application/json",
"X-API-Key": randomApiKey,
}, nil, subscraping.BasicAuth{})
@@ -72,19 +75,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
} else if resp.StatusCode != 200 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)}
} else if resp1.StatusCode != 200 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)}
s.errors++
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
if err := resp1.Body.Close(); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
}
}()
body, err := io.ReadAll(resp.Body)
body, err := io.ReadAll(resp1.Body)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")}
s.errors++
@@ -120,7 +123,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
// Pick an API key
randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name())
resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{
s.requests++
resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{
"accept": "application/json",
"X-API-Key": randomApiKey,
"Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{})
@@ -130,20 +134,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
if err := resp2.Body.Close(); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
}
}()
body, err = io.ReadAll(resp.Body)
body, err = io.ReadAll(resp2.Body)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")}
s.errors++
return
}
if resp.StatusCode == 429 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)}
if resp2.StatusCode == 429 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)}
s.errors++
return
}
@@ -158,10 +162,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, item := range data {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain,
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain}:
s.results++
}
s.results++
}
}()
@@ -182,8 +188,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -194,6 +204,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/onyphe/onyphe.go
+318
View File
@@ -0,0 +1,318 @@
// Package onyphe logic
package onyphe
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type OnypheResponse struct {
Error int `json:"error"`
Results []Result `json:"results"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int `json:"total"`
MaxPage int `json:"max_page"`
}
type Result struct {
Subdomains []string `json:"subdomains"`
Hostname string `json:"hostname"`
Forward string `json:"forward"`
Reverse string `json:"reverse"`
Host string `json:"host"`
Domain string `json:"domain"`
}
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey}
page := 1
pageSize := 1000
for {
select {
case <-ctx.Done():
return
default:
}
var resp *http.Response
var err error
urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d",
url.QueryEscape("category:resolver domain:"+domain), page, pageSize)
s.requests++
resp, err = session.Get(ctx, urlWithQuery, "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var respOnyphe OnypheResponse
err = json.NewDecoder(resp.Body).Decode(&respOnyphe)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)
for _, record := range respOnyphe.Results {
select {
case <-ctx.Done():
return
default:
}
for _, subdomain := range record.Subdomains {
if subdomain != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
}
if record.Hostname != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
s.results++
}
if record.Forward != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward}
s.results++
}
if record.Reverse != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse}
s.results++
}
}
if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage {
break
}
page++
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "onyphe"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
type OnypheResponseRaw struct {
Error int `json:"error"`
Results []Result `json:"results"`
Page json.RawMessage `json:"page"`
PageSize json.RawMessage `json:"page_size"`
Total json.RawMessage `json:"total"`
MaxPage json.RawMessage `json:"max_page"`
}
func (o *OnypheResponse) UnmarshalJSON(data []byte) error {
var raw OnypheResponseRaw
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
o.Error = raw.Error
o.Results = raw.Results
if pageStr := string(raw.Page); pageStr != "" {
if page, err := strconv.Atoi(pageStr); err == nil {
o.Page = page
} else {
var pageStrQuoted string
if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil {
if page, err := strconv.Atoi(pageStrQuoted); err == nil {
o.Page = page
}
}
}
}
if pageSizeStr := string(raw.PageSize); pageSizeStr != "" {
if pageSize, err := strconv.Atoi(pageSizeStr); err == nil {
o.PageSize = pageSize
} else {
var pageSizeStrQuoted string
if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil {
if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil {
o.PageSize = pageSize
}
}
}
}
if totalStr := string(raw.Total); totalStr != "" {
if total, err := strconv.Atoi(totalStr); err == nil {
o.Total = total
} else {
var totalStrQuoted string
if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil {
if total, err := strconv.Atoi(totalStrQuoted); err == nil {
o.Total = total
}
}
}
}
if maxPageStr := string(raw.MaxPage); maxPageStr != "" {
if maxPage, err := strconv.Atoi(maxPageStr); err == nil {
o.MaxPage = maxPage
} else {
var maxPageStrQuoted string
if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil {
if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil {
o.MaxPage = maxPage
}
}
}
}
return nil
}
type ResultRaw struct {
Subdomains json.RawMessage `json:"subdomains"`
Hostname json.RawMessage `json:"hostname"`
Forward json.RawMessage `json:"forward"`
Reverse json.RawMessage `json:"reverse"`
Host json.RawMessage `json:"host"`
Domain json.RawMessage `json:"domain"`
}
func (r *Result) UnmarshalJSON(data []byte) error {
var raw ResultRaw
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
var subdomains []string
if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil {
r.Subdomains = subdomains
} else {
var subdomainStr string
if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil {
r.Subdomains = []string{subdomainStr}
}
}
if len(raw.Hostname) > 0 {
var hostnameStr string
if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil {
r.Hostname = hostnameStr
} else {
var hostnameArr []string
if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 {
r.Hostname = hostnameArr[0]
}
}
}
if len(raw.Forward) > 0 {
_ = json.Unmarshal(raw.Forward, &r.Forward)
}
if len(raw.Reverse) > 0 {
_ = json.Unmarshal(raw.Reverse, &r.Reverse)
}
if len(raw.Host) > 0 {
var hostStr string
if err := json.Unmarshal(raw.Host, &hostStr); err == nil {
r.Host = hostStr
} else {
var hostArr []string
if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 {
r.Host = hostArr[0]
}
}
}
if len(raw.Domain) > 0 {
var domainStr string
if err := json.Unmarshal(raw.Domain, &domainStr); err == nil {
r.Domain = domainStr
} else {
var domainArr []string
if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 {
r.Domain = domainArr[0]
}
}
}
return nil
}
pkg/subscraping/sources/profundis/profundis.go
+135
View File
@@ -0,0 +1,135 @@
// Package profundis logic
package profundis
import (
"bufio"
"bytes"
"context"
"encoding/json"
"strings"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
requestBody, err := json.Marshal(map[string]string{"domain": domain})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
headers := map[string]string{
"Content-Type": "application/json",
"X-API-KEY": randomApiKey,
"Accept": "text/event-stream",
}
s.requests++
resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "",
headers, bytes.NewReader(requestBody))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
}
}()
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
return
default:
}
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: line}:
s.results++
}
}
if err := scanner.Err(); err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
}
}()
return results
}
func (s *Source) Name() string {
return "profundis"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/pugrecon/pugrecon.go
+15 -4
View File
@@ -32,6 +32,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -71,6 +73,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
apiURL := "https://pugrecon.com/api/v1/domains"
s.requests++
resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -106,10 +109,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range response.Results {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name,
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name}:
s.results++
}
s.results++
}
}()
@@ -131,9 +136,14 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
// KeyRequirement returns the API key requirement level for this source.
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
// NeedsKey returns true as this source requires an API key.
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
// AddApiKeys adds the API keys for the source.
@@ -148,5 +158,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/quake/quake.go
+19 -1
View File
@@ -36,6 +36,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -63,7 +65,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
var totalResults = -1
for {
select {
case <-ctx.Done():
return
default:
}
var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start)
s.requests++
resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
"Content-Type": "application/json", "X-QuakeToken": randomApiKey,
}, bytes.NewReader(requestBody))
@@ -97,6 +105,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, quakeDomain := range response.Data {
select {
case <-ctx.Done():
return
default:
}
subdomain := quakeDomain.Service.HTTP.Host
if strings.ContainsAny(subdomain, "暂无权限") {
continue
@@ -129,8 +142,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -143,5 +160,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/rapiddns/rapiddns.go
+20 -3
View File
@@ -19,6 +19,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -36,6 +38,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
page := 1
maxPages := 1
for {
select {
case <-ctx.Done():
return
default:
}
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -56,8 +64,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
src := string(body)
for _, subdomain := range session.Extractor.Extract(src) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
if maxPages == 1 {
@@ -93,8 +105,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -106,5 +122,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/reconcloud/reconcloud.go
+15 -3
View File
@@ -29,6 +29,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -63,8 +66,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if len(response.CloudAssetsList) > 0 {
for _, cloudAsset := range response.CloudAssetsList {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}:
s.results++
}
}
}
}()
@@ -85,8 +92,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -98,5 +109,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/reconeer/reconeer.go
+116
View File
@@ -0,0 +1,116 @@
package reconeer
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type response struct {
Subdomains []subdomain `json:"subdomains"`
}
type subdomain struct {
Subdomain string `json:"subdomain"`
}
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
headers := map[string]string{
"Accept": "application/json",
}
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey != "" {
headers["X-API-KEY"] = randomApiKey
}
apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain)
s.requests++
resp, err := session.Get(ctx, apiURL, "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
defer session.DiscardHTTPResponse(resp)
if resp.StatusCode != 200 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)}
s.errors++
return
}
var responseData response
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(&responseData)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
for _, result := range responseData.Subdomains {
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}:
s.results++
}
}
}()
return results
}
func (s *Source) Name() string {
return "reconeer"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.OptionalKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/redhuntlabs/redhuntlabs.go
+27 -5
View File
@@ -28,6 +28,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
pageSize := 1000
go func() {
defer func(startTime time.Time) {
@@ -56,6 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1]
requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"}
getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize)
s.requests++
resp, err := session.Get(ctx, getUrl, "", requestHeaders)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)}
@@ -76,7 +79,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if response.Metadata.ResultCount > pageSize {
totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize
for page := 1; page <= totalPages; page++ {
select {
case <-ctx.Done():
return
default:
}
getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize)
s.requests++
resp, err := session.Get(ctx, getUrl, "", requestHeaders)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)}
@@ -96,14 +105,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
for _, subdomain := range response.Subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
} else {
for _, subdomain := range response.Subdomains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
@@ -123,8 +140,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -137,5 +158,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/riddler/riddler.go
+22 -3
View File
@@ -15,6 +15,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -22,6 +23,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -29,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -39,13 +42,24 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
default:
}
line := scanner.Text()
if line == "" {
continue
}
for _, subdomain := range session.Extractor.Extract(line) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
session.DiscardHTTPResponse(resp)
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
session.DiscardHTTPResponse(resp)
@@ -67,8 +81,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -80,5 +98,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/robtex/robtext.go
+16 -3
View File
@@ -62,6 +62,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, result := range ips {
select {
case <-ctx.Done():
return
default:
}
if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord {
domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers)
if err != nil {
@@ -70,8 +75,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
for _, result := range domains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}:
s.results++
}
}
}
}
@@ -122,8 +131,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
pkg/subscraping/sources/rsecloud/rsecloud.go
+20 -3
View File
@@ -24,6 +24,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -50,6 +52,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
fetchSubdomains := func(endpoint string) {
page := 1
for {
select {
case <-ctx.Done():
return
default:
}
s.requests++
resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -68,8 +76,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range rseCloudResponse.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
if page >= rseCloudResponse.TotalPages {
@@ -99,8 +111,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -113,5 +129,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/securitytrails/securitytrails.go
+27 -3
View File
@@ -31,6 +31,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -56,18 +58,26 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey}
for {
select {
case <-ctx.Done():
return
default:
}
var resp *http.Response
var err error
if scrollId == "" {
var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain)
s.requests++
resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "",
headers, bytes.NewReader(requestBody))
} else {
s.requests++
resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers)
}
if err != nil && ptr.Safe(resp).StatusCode == 403 {
s.requests++
resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers)
}
@@ -90,11 +100,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
for _, record := range securityTrailsResponse.Records {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}:
s.results++
}
}
for _, subdomain := range securityTrailsResponse.Subdomains {
select {
case <-ctx.Done():
return
default:
}
if strings.HasSuffix(subdomain, ".") {
subdomain += domain
} else {
@@ -128,8 +147,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -142,5 +165,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/shodan/shodan.go
+19 -1
View File
@@ -17,6 +17,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -48,8 +50,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
page := 1
for {
select {
case <-ctx.Done():
return
default:
}
searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page)
s.requests++
resp, err := session.SimpleGet(ctx, searchURL)
if err != nil {
session.DiscardHTTPResponse(resp)
@@ -75,6 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, data := range response.Subdomains {
select {
case <-ctx.Done():
return
default:
}
value := fmt.Sprintf("%s.%s", data, response.Domain)
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Subdomain, Value: value,
@@ -105,8 +118,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -117,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/sitedossier/sitedossier.go
+15 -3
View File
@@ -23,6 +23,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -50,6 +52,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba
default:
}
s.requests++
resp, err := session.SimpleGet(ctx, baseURL)
isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound
if err != nil && !isnotfound {
@@ -70,8 +73,12 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba
src := string(body)
for _, subdomain := range session.Extractor.Extract(src) {
results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
match := reNext.FindStringSubmatch(src)
@@ -93,8 +100,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -106,5 +117,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/submd/submd.go
+14 -4
View File
@@ -15,12 +15,14 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -28,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := s.fetch(ctx, domain, session)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -41,8 +44,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for sc.Scan() {
if line := sc.Text(); line != "" {
for _, sub := range session.Extractor.Extract(line) {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub}:
s.results++
}
}
}
}
@@ -70,13 +77,16 @@ func (s *Source) fetch(ctx context.Context, domain string, session *subscraping.
func (s *Source) Name() string { return "submd" }
func (s *Source) IsDefault() bool { return true }
func (s *Source) HasRecursiveSupport() bool { return false }
func (s *Source) NeedsKey() bool { return false }
func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys }
func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.OptionalKey }
func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey }
func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys }
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
}
}
pkg/subscraping/sources/thc/thc.go
+135
View File
@@ -0,0 +1,135 @@
// Package thc logic
package thc
import (
"bytes"
"context"
"encoding/json"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type response struct {
Domains []struct {
Domain string `json:"domain"`
} `json:"domains"`
NextPageState string `json:"next_page_state"`
}
// Source is the passive scraping agent
type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
type requestBody struct {
Domain string `json:"domain"`
PageState string `json:"page_state"`
Limit int `json:"limit"`
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
var pageState string
headers := map[string]string{"Content-Type": "application/json"}
apiURL := "https://ip.thc.org/api/v1/lookup/subdomains"
for {
reqBody := requestBody{
Domain: domain,
PageState: pageState,
Limit: 1000,
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
s.requests++
resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var thcResponse response
err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse)
session.DiscardHTTPResponse(resp)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
return
}
for _, domainRecord := range thcResponse.Domains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain}
s.results++
}
pageState = thcResponse.NextPageState
if pageState == "" {
break
}
}
}()
return results
}
// Name returns the name of the source
func (s *Source) Name() string {
return "thc"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
// No API keys needed for THC
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/threatbook/threatbook.go
+15 -3
View File
@@ -30,6 +30,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -51,6 +53,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -87,8 +90,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
if total > 0 {
for _, subdomain := range response.Data.SubDomains.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}()
@@ -109,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -123,5 +134,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/threatcrowd/threatcrowd.go
+16 -3
View File
@@ -23,6 +23,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run queries the ThreatCrowd API for the given domain and returns found subdomains.
@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func(startTime time.Time) {
defer func() {
@@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.Client.Do(req)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -80,8 +83,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for _, subdomain := range tcResponse.Subdomains {
if subdomain != "" {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}(time.Now())
@@ -104,9 +111,14 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
// KeyRequirement returns the API key requirement level for this source.
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
// NeedsKey indicates if the source requires an API key.
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
// AddApiKeys is a no-op since ThreatCrowd does not require an API key.
@@ -118,5 +130,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/threatminer/threatminer.go
+15 -3
View File
@@ -22,6 +22,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -36,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -55,8 +58,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range data.Results {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}()
@@ -76,8 +83,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -89,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/urlscan/urlscan.go
+215
View File
@@ -0,0 +1,215 @@
// Package urlscan logic
package urlscan
import (
"context"
"fmt"
"net/url"
"strings"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
const (
// baseURL is the URLScan API base URL
baseURL = "https://urlscan.io/api/v1/search/"
// maxPages is the maximum number of pages to fetch
maxPages = 5
// maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer)
maxPerPage = 100
)
// response represents the URLScan API response structure
type response struct {
Results []struct {
Task struct {
Domain string `json:"domain"`
URL string `json:"url"`
} `json:"task"`
Page struct {
Domain string `json:"domain"`
URL string `json:"url"`
} `json:"page"`
Sort []interface{} `json:"sort"`
} `json:"results"`
HasMore bool `json:"has_more"`
Total int `json:"total"`
}
// Source is the passive scraping agent
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
s.skipped = false
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
headers := map[string]string{"api-key": randomApiKey}
// Search with wildcard to get more subdomain results
s.enumerate(ctx, domain, headers, session, results)
}()
return results
}
// enumerate performs the actual enumeration with pagination
func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) {
var searchAfter string
currentPage := 0
for {
select {
case <-ctx.Done():
return
default:
}
if currentPage >= maxPages {
break
}
// Build search URL
searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage)
if searchAfter != "" {
searchURL += "&search_after=" + url.QueryEscape(searchAfter)
}
s.requests++
resp, err := session.Get(ctx, searchURL, "", headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var data response
err = jsoniter.NewDecoder(resp.Body).Decode(&data)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)
// Process results - extract subdomains from multiple fields
for _, result := range data.Results {
candidates := []string{
result.Task.Domain,
result.Page.Domain,
}
// Also extract from URLs if present
if result.Task.URL != "" {
if u, err := url.Parse(result.Task.URL); err == nil {
candidates = append(candidates, u.Hostname())
}
}
if result.Page.URL != "" {
if u, err := url.Parse(result.Page.URL); err == nil {
candidates = append(candidates, u.Hostname())
}
}
for _, candidate := range candidates {
if candidate == "" {
continue
}
for _, subdomain := range session.Extractor.Extract(candidate) {
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}
// Check pagination conditions
if !data.HasMore || len(data.Results) == 0 {
break
}
// Get sort value for next page
lastResult := data.Results[len(data.Results)-1]
if len(lastResult.Sort) == 0 {
break
}
// Build search_after parameter
sortValues := make([]string, len(lastResult.Sort))
for i, v := range lastResult.Sort {
switch val := v.(type) {
case float64:
sortValues[i] = fmt.Sprintf("%.0f", val)
default:
sortValues[i] = fmt.Sprintf("%v", v)
}
}
searchAfter = strings.Join(sortValues, ",")
currentPage++
}
}
// Name returns the name of the source
func (s *Source) Name() string {
return "urlscan"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
}
pkg/subscraping/sources/virustotal/virustotal.go
+20 -3
View File
@@ -30,6 +30,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -51,10 +53,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
var cursor = ""
for {
select {
case <-ctx.Done():
return
default:
}
var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain)
if cursor != "" {
url = fmt.Sprintf("%s&cursor=%s", url, cursor)
}
s.requests++
resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey})
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -78,8 +86,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
for _, subdomain := range data.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}:
s.results++
}
}
cursor = data.Meta.Cursor
if cursor == "" {
@@ -104,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool {
return true
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -116,6 +132,7 @@ func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
Requests: s.requests,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
}
pkg/subscraping/sources/waybackarchive/waybackarchive.go
+20 -4
View File
@@ -17,6 +17,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
}
// Run function returns all subdomains found with the service
@@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
close(results)
}(time.Now())
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -43,19 +46,27 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
select {
case <-ctx.Done():
return
default:
}
line := scanner.Text()
if line == "" {
continue
}
line, _ = url.QueryUnescape(line)
for _, subdomain := range session.Extractor.Extract(line) {
// fix for triple encoded URL
subdomain = strings.ToLower(subdomain)
subdomain = strings.TrimPrefix(subdomain, "25")
subdomain = strings.TrimPrefix(subdomain, "2f")
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}:
s.results++
}
}
}
}()
@@ -76,8 +87,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.NoKey
}
func (s *Source) NeedsKey() bool {
return false
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(_ []string) {
@@ -89,5 +104,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Requests: s.requests,
}
}
pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go
+15 -3
View File
@@ -33,6 +33,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -41,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -54,6 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
return
}
s.requests++
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -74,8 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
session.DiscardHTTPResponse(resp)
for _, record := range data.Result.Records {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}:
s.results++
}
}
}()
@@ -95,8 +102,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -109,5 +120,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/windvane/windvane.go
+162
View File
@@ -0,0 +1,162 @@
// Package windvane logic
package windvane
import (
"bytes"
"context"
"encoding/json"
"net/http"
"strconv"
"time"
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
)
type response struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data responseData `json:"data"`
}
type responseData struct {
List []domainEntry `json:"list"`
PageResponse pageInfo `json:"page_response"`
}
type domainEntry struct {
Domain string `json:"domain"`
}
type pageInfo struct {
Total string `json:"total"`
Count string `json:"count"`
TotalPage string `json:"total_page"`
}
type Source struct {
apiKeys []string
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
s.timeTaken = time.Since(startTime)
close(results)
}(time.Now())
randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
if randomApiKey == "" {
s.skipped = true
return
}
headers := map[string]string{"Content-Type": "application/json", "X-Api-Key": randomApiKey}
page := 1
count := 1000
for {
select {
case <-ctx.Done():
return
default:
}
var resp *http.Response
var err error
requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}})
s.requests++
resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain",
"", headers, bytes.NewReader(requestBody))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
defer session.DiscardHTTPResponse(resp)
var windvaneResponse response
err = json.NewDecoder(resp.Body).Decode(&windvaneResponse)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
for _, record := range windvaneResponse.Data.List {
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}:
s.results++
}
}
pageInfo := windvaneResponse.Data.PageResponse
var totalRecords, recordsPerPage int
if totalRecords, err = strconv.Atoi(pageInfo.Total); err != nil {
break
}
if recordsPerPage, err = strconv.Atoi(pageInfo.Count); err != nil {
break
}
if (page-1)*recordsPerPage >= totalRecords {
break
}
page++
}
}()
return results
}
func (s *Source) Name() string {
return "windvane"
}
func (s *Source) IsDefault() bool {
return true
}
func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
s.apiKeys = keys
}
func (s *Source) Statistics() subscraping.Statistics {
return subscraping.Statistics{
Errors: s.errors,
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go
+20 -3
View File
@@ -27,6 +27,7 @@ type Source struct {
timeTaken time.Duration
errors int
results int
requests int
skipped bool
}
@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
results := make(chan subscraping.Result)
s.errors = 0
s.results = 0
s.requests = 0
go func() {
defer func(startTime time.Time) {
@@ -63,7 +65,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}
var pages = 1
for currentPage := 1; currentPage <= pages; currentPage++ {
select {
case <-ctx.Done():
return
default:
}
api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage)
s.requests++
resp, err := session.Get(ctx, api, "", headers)
isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
if err != nil {
@@ -87,8 +95,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
_ = resp.Body.Close()
pages = int(res.Total/1000) + 1
for _, r := range res.List {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}
s.results++
select {
case <-ctx.Done():
return
case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}:
s.results++
}
}
}
}()
@@ -109,8 +121,12 @@ func (s *Source) HasRecursiveSupport() bool {
return false
}
func (s *Source) KeyRequirement() subscraping.KeyRequirement {
return subscraping.RequiredKey
}
func (s *Source) NeedsKey() bool {
return true
return s.KeyRequirement() == subscraping.RequiredKey
}
func (s *Source) AddApiKeys(keys []string) {
@@ -123,5 +139,6 @@ func (s *Source) Statistics() subscraping.Statistics {
Results: s.results,
TimeTaken: s.timeTaken,
Skipped: s.skipped,
Requests: s.requests,
}
}
pkg/subscraping/types.go
+19 -2
View File
@@ -16,7 +16,8 @@ const (
)
type CustomRateLimit struct {
Custom mapsutil.SyncLockMap[string, uint]
Custom mapsutil.SyncLockMap[string, uint]
CustomDuration mapsutil.SyncLockMap[string, time.Duration]
}
// BasicAuth request's Authorization header
@@ -28,11 +29,21 @@ type BasicAuth struct {
// Statistics contains statistics about the scraping process
type Statistics struct {
TimeTaken time.Duration
Requests int
Errors int
Results int
Skipped bool
}
// KeyRequirement represents the API key requirement level for a source
type KeyRequirement int
const (
NoKey KeyRequirement = iota
OptionalKey
RequiredKey
)
// Source is an interface inherited by each passive source
type Source interface {
// Run takes a domain as argument and a session object
@@ -52,7 +63,11 @@ type Source interface {
// not just root domains.
HasRecursiveSupport() bool
// NeedsKey returns true if the source requires an API key
// KeyRequirement returns the API key requirement level for this source
KeyRequirement() KeyRequirement
// NeedsKey returns true if the source requires an API key.
// Deprecated: Use KeyRequirement() instead for more granular control.
NeedsKey() bool
AddApiKeys([]string)
@@ -75,6 +90,8 @@ type Session struct {
Client *http.Client
// Rate limit instance
MultiRateLimiter *ratelimit.MultiLimiter
// Timeout is the timeout in seconds for requests
Timeout int
}
// Result is a result structure returned by a source