diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..ff63af57 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Proposed changes + + + +### Proof + + + +## Checklist + + + +- [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch +- [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added necessary documentation (if appropriate) \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba33d912..63f4b632 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,7 +8,7 @@ updates: # Maintain dependencies for go modules - package-ecosystem: "gomod" - directory: "v2/" + directory: "/" schedule: interval: "weekly" target-branch: "dev" @@ -45,4 +45,4 @@ updates: # prefix: "chore" # include: "scope" # labels: -# - "Type: Maintenance" \ No newline at end of file +# - "Type: Maintenance" diff --git a/.github/release.yml b/.github/release.yml index 1073bdc8..550dbea5 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -8,10 +8,10 @@ changelog: - "Type: Enhancement" - title: 🐞 Bug Fixes labels: - - "Type: Bug" + - "Type: Bug" - title: 🔨 Maintenance labels: - - "Type: Maintenance" + - "Type: Maintenance" - title: Other Changes labels: - "*" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 80298e48..939fc835 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,15 +19,9 @@ jobs: if: "${{ !endsWith(github.actor, '[bot]') }}" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - with: - go-version-file: go.mod - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8 - with: - version: latest - args: --timeout 5m + - uses: projectdiscovery/actions/golangci-lint/v2@v1 build: name: Test Builds @@ -35,12 +29,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: projectdiscovery/actions/setup/go@v1 - with: - go-version-file: go.mod - run: go build ./... - name: Run tests @@ -58,7 +50,6 @@ jobs: FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} - HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} @@ -66,10 +57,11 @@ jobs: SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v3 with: timeout_seconds: 360 max_attempts: 3 @@ -80,3 +72,4 @@ jobs: - name: Run Example run: go run . + working-directory: examples diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 927b1cc1..516d067b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,14 +7,15 @@ on: - '**.go' - '**.mod' +permissions: + actions: read + contents: read + security-events: write + jobs: analyze: name: Analyze runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write strategy: fail-fast: false @@ -23,17 +24,17 @@ jobs: # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v6 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + - name: Autobuild + uses: github/codeql-action/autobuild@v4 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yml similarity index 55% rename from .github/workflows/compat-checks.yaml rename to .github/workflows/compat-checks.yml index 8efb822d..7f6e9bc8 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yml @@ -6,15 +6,15 @@ on: branches: - dev +permissions: + contents: write + jobs: check: if: github.actor == 'dependabot[bot]' runs-on: ubuntu-latest - permissions: - contents: write steps: - - uses: actions/checkout@v4 - - uses: projectdiscovery/actions/setup/go/compat-checks@master + - uses: actions/checkout@v6 + - uses: projectdiscovery/actions/setup/go/compat-checks@v1 with: - go-version-file: 'go.mod' - + go-version-file: go.mod diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml index 84b26e1f..a21b9338 100644 --- a/.github/workflows/dep-auto-merge.yml +++ b/.github/workflows/dep-auto-merge.yml @@ -16,11 +16,11 @@ jobs: runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: token: ${{ secrets.DEPENDABOT_PAT }} - uses: ahmadnassri/action-dependabot-auto-merge@v2 with: github-token: ${{ secrets.DEPENDABOT_PAT }} - target: all \ No newline at end of file + target: all diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 4409dcc3..582c3253 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Get Github tag id: meta @@ -20,19 +20,19 @@ jobs: curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64,linux/arm diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index ea68f5d0..55f340d0 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -11,20 +11,16 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - - - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v3 + uses: actions/checkout@v6 with: - args: "release --clean" - version: latest + fetch-depth: 0 + + - uses: projectdiscovery/actions/setup/go@v1 + + - name: "Create release on GitHub" + uses: projectdiscovery/actions/goreleaser@v1 + with: + release: true env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 1c8fbbf8..6c5b6cc9 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -12,17 +12,11 @@ jobs: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" - uses: actions/checkout@v3 - with: + uses: actions/checkout@v6 + with: fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: 1.21.x - + - uses: projectdiscovery/actions/setup/go@v1 + - name: release test - uses: goreleaser/goreleaser-action@v4 - with: - args: "release --clean --snapshot" - version: latest + uses: projectdiscovery/actions/goreleaser@v1 diff --git a/.gitignore b/.gitignore index b0dd47ba..155676bf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ vendor/ .idea .devcontainer .vscode -dist \ No newline at end of file +dist +/subfinder \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 867267d2..151920f6 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -5,31 +5,31 @@ before: - go mod tidy builds: -- env: - - CGO_ENABLED=0 - goos: - - windows - - linux - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 + - env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - '386' + - arm + - arm64 - ignore: - - goos: darwin - goarch: '386' - - goos: windows - goarch: 'arm' + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' - binary: '{{ .ProjectName }}' - main: cmd/subfinder/main.go + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go archives: -- formats: - - zip - name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' + - formats: + - zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256 diff --git a/README.md b/README.md index 399998af..dad54507 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,13 @@ OPTIMIZATION: -max-time int minutes to wait for enumeration results (default 10) ``` +## Environment Variables + +Subfinder supports environment variables to specify custom paths for configuration files: + +- `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`) +- `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`) + # Installation `subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: diff --git a/go.mod b/go.mod index 436e17e8..79fb7f97 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,13 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 - github.com/projectdiscovery/dnsx v1.2.2 + github.com/projectdiscovery/dnsx v1.2.3 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/ratelimit v0.0.81 - github.com/projectdiscovery/retryablehttp-go v1.0.115 - github.com/projectdiscovery/utils v0.4.21 + github.com/projectdiscovery/gologger v1.1.68 + github.com/projectdiscovery/ratelimit v0.0.84 + github.com/projectdiscovery/utils v0.10.0 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 gopkg.in/yaml.v3 v3.0.1 @@ -28,28 +27,28 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect - github.com/STARRY-S/zip v0.2.1 // indirect + github.com/STARRY-S/zip v0.2.3 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/sevenzip v1.6.1 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/djherbis/times v1.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.20.4 // indirect + github.com/gaissmai/bart v0.26.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -57,39 +56,41 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/archives v0.1.0 // indirect + github.com/mholt/archives v0.1.5 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect - github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/nwaples/rardecode/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.23 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.24 // indirect - github.com/projectdiscovery/fastdialer v0.4.1 // indirect - github.com/projectdiscovery/hmap v0.0.90 // indirect - github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.16 // indirect - github.com/refraction-networking/utls v1.7.0 // indirect + github.com/projectdiscovery/cdncheck v1.2.28 // indirect + github.com/projectdiscovery/fastdialer v0.5.4 // indirect + github.com/projectdiscovery/hmap v0.0.100 // indirect + github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect + github.com/projectdiscovery/networkpolicy v0.1.34 // indirect + github.com/projectdiscovery/retryablehttp-go v1.3.6 // indirect + github.com/refraction-networking/utls v1.8.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/sorairolake/lzip-go v0.3.8 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect @@ -100,26 +101,25 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect - github.com/weppos/publicsuffix-go v0.30.1 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect + github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect - github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect + github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/mod v0.22.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.30.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.29.0 // indirect - gopkg.in/djherbis/times.v1 v1.3.0 // indirect + golang.org/x/tools v0.39.0 // indirect ) require ( @@ -132,7 +132,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.102 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect + github.com/projectdiscovery/retryabledns v1.0.114 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index 2aeebb58..a7b84af3 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2piv github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= -github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= +github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= +github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= @@ -39,8 +39,8 @@ github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46 github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -55,8 +55,8 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= -github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= +github.com/bodgit/sevenzip v1.6.1/go.mod h1:GVoYQbEVbOGT8n2pfqCIMRUaRjQ8F9oSqoBEqZh5fQ8= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -76,8 +76,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -89,6 +87,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -103,8 +103,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= -github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= +github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0= +github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -137,11 +137,11 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= -github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -162,11 +162,6 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -183,8 +178,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -200,6 +195,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -212,12 +209,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= -github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= +github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= +github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= +github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= +github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc= +github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A= +github.com/minio/minlz v1.0.1/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -231,8 +232,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= -github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= -github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= +github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -243,8 +244,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.23 h1:oJE7T90aYBGtFNrI8+KbETnPymobAhzRrR8Mu8n1yfU= +github.com/pierrec/lz4/v4 v4.1.23/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -255,37 +256,37 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs= -github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.2.28 h1:EhOK+mi0bfLb+CdiJaFYmMMuX7xu0U820e5GFbP2usc= +github.com/projectdiscovery/cdncheck v1.2.28/go.mod h1:Y1KQmACY+AifbuPX/W7o8lWssiWmAZ5d/KG8qkmFm9I= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= -github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= -github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= -github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4= -github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I= +github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= +github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= +github.com/projectdiscovery/fastdialer v0.5.4 h1:+0oesDDqZcIPE5bNDmm/Xm9Xm3yjnhl4xwP+h5D1TE4= +github.com/projectdiscovery/fastdialer v0.5.4/go.mod h1:KCzt6WnSAj9umiUBRCaC0EJSEyeshxDoowfwjxodmQw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= -github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= -github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4= -github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= -github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI= -github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U= -github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= -github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= -github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI= -github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E= -github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E= -github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA= -github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q= -github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I= +github.com/projectdiscovery/gologger v1.1.68 h1:KfdIO/3X7BtHssWZuqhxPZ+A946epCCx2cz+3NnRAnU= +github.com/projectdiscovery/gologger v1.1.68/go.mod h1:Xae0t4SeqJVa0RQGK9iECx/+HfXhvq70nqOQp2BuW+o= +github.com/projectdiscovery/hmap v0.0.100 h1:DBZ3Req9lWf4P1YC9PRa4eiMvLY0Uxud43NRBcocPfs= +github.com/projectdiscovery/hmap v0.0.100/go.mod h1:2O06pR8pHOP9wSmxAoxuM45U7E+UqOqOdlSIeddM0bA= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q= +github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/networkpolicy v0.1.34 h1:TRwNbgMwdx3NC190TKSLwtTvr0JAIZAlnWkOhW0yBME= +github.com/projectdiscovery/networkpolicy v0.1.34/go.mod h1:GJ20E7fJoA2vk8ZBSa1Cvc5WyP8RxglF5bZmYgK8jag= +github.com/projectdiscovery/ratelimit v0.0.84 h1:RzaO+OBGABI5G0e045TLwedV7K6o0dTOy9pxYmH+tbU= +github.com/projectdiscovery/ratelimit v0.0.84/go.mod h1:U0OGcBJk6+Hz9nERWZVN4s8XL3pF9XxQJwPlEbF4pr0= +github.com/projectdiscovery/retryabledns v1.0.114 h1:COyNKzhA7oa3C/1639WRXeXsKrUJx06paVbN64IHZ3E= +github.com/projectdiscovery/retryabledns v1.0.114/go.mod h1:+DyanDr8naxQ2dRO9c4Ezo3NHHXhz8L0tTSRYWhiwyA= +github.com/projectdiscovery/retryablehttp-go v1.3.6 h1:dLb0/YVX+oX70gpWxN5GXT8pCKpn8fdXfwOq2TsXxNY= +github.com/projectdiscovery/retryablehttp-go v1.3.6/go.mod h1:tKVxmL4ixWy1MjYk5GJvFL0Cp10fnQgSp2F6bSBEypI= +github.com/projectdiscovery/utils v0.10.0 h1:E3nMm0h3LWt2bbnpRd8Whyj/y0DrMJKYx2zV8gqFDF0= +github.com/projectdiscovery/utils v0.10.0/go.mod h1:FL0cQdg3oBMtJdmbBrfLd5i73syNxpkbKO9tivQ0+rI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= -github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= +github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= +github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -307,13 +308,17 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= -github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sorairolake/lzip-go v0.3.8 h1:j5Q2313INdTA80ureWYRhX+1K78mUXfMoPZCw/ivWik= +github.com/sorairolake/lzip-go v0.3.8/go.mod h1:JcBqGMV0frlxwrsE9sMWXDjqn3EeVf0/54YPsw66qkU= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -322,12 +327,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= @@ -359,12 +362,12 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= -github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= +github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2 h1:LiQSn5u8Nc6V/GixI+SWxt+YkNIyfKIlkVRULSw2Zt0= +github.com/weppos/publicsuffix-go v0.50.3-0.20260104170930-90713dec78f2/go.mod h1:CbQCKDtXF8UcT7hrxeMa0MDjwhpOI9iYOU7cfq+yo8k= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -387,8 +390,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk= +github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= @@ -409,14 +412,15 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -446,8 +450,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -471,15 +478,17 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= @@ -491,8 +500,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -518,6 +530,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -527,17 +540,25 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -549,9 +570,12 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -582,8 +606,10 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -624,13 +650,11 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= -gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/pkg/passive/passive.go b/pkg/passive/passive.go index 199686bd..c00a024d 100644 --- a/pkg/passive/passive.go +++ b/pkg/passive/passive.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + mapsutil "github.com/projectdiscovery/utils/maps" ) type EnumerationOptions struct { @@ -65,11 +66,15 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p for _, runner := range a.sources { wg.Add(1) go func(source subscraping.Source) { + defer wg.Done() ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { - results <- resp + select { + case <-ctx.Done(): + return + case results <- resp: + } } - wg.Done() }(runner) } wg.Wait() @@ -81,14 +86,21 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error - for _, source := range a.sources { - var rl uint - if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { - rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) + if rateLimit == nil { + rateLimit = &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, } + } + for _, source := range a.sources { + rl, duration := resolveSourceRateLimit(globalRateLimit, rateLimit, source.Name()) if rl > 0 { - multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) + multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, duration) } else { multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) } @@ -100,6 +112,30 @@ func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, return multiRateLimiter, err } +// resolveSourceRateLimit returns the effective rate limit and duration for a source. +// Priority: per-source custom limit > global -rl limit > unlimited. +// Duration comes from -rls (e.g. hackertarget=2/m → 2 per minute), defaulting to per-second. +func resolveSourceRateLimit(globalRateLimit int, rateLimit *subscraping.CustomRateLimit, sourceName string) (uint, time.Duration) { + duration := time.Second // default: requests per second + + sourceLower := strings.ToLower(sourceName) + if sourceRL, ok := rateLimit.Custom.Get(sourceLower); ok { + rl := sourceRateLimitOrDefault(uint(max(globalRateLimit, 0)), sourceRL) + // Use per-source duration from -rls if set (e.g. "2/m" → time.Minute) + if d, ok := rateLimit.CustomDuration.Get(sourceLower); ok && d > 0 { + duration = d + } + return rl, duration + } + + // No per-source limit: fall back to global -rl + if globalRateLimit > 0 { + return uint(globalRateLimit), duration + } + + return 0, duration +} + func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { if sourceRateLimit > 0 { return sourceRateLimit diff --git a/pkg/passive/ratelimit_test.go b/pkg/passive/ratelimit_test.go new file mode 100644 index 00000000..36c929e1 --- /dev/null +++ b/pkg/passive/ratelimit_test.go @@ -0,0 +1,147 @@ +package passive + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + mapsutil "github.com/projectdiscovery/utils/maps" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +func newCustomRateLimit(counts map[string]uint, durations map[string]time.Duration) *subscraping.CustomRateLimit { + crl := &subscraping.CustomRateLimit{ + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, + } + for k, v := range counts { + _ = crl.Custom.Set(k, v) + } + for k, d := range durations { + _ = crl.CustomDuration.Set(k, d) + } + return crl +} + +func TestResolveSourceRateLimit_PerSourceOverride(t *testing.T) { + // Per-source limit should override global + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Minute, dur) +} + +func TestResolveSourceRateLimit_GlobalFallback(t *testing.T) { + // Sources without per-source limit should use global -rl + crl := newCustomRateLimit(nil, nil) + + rl, dur := resolveSourceRateLimit(10, crl, "crtsh") + assert.Equal(t, uint(10), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_NoLimit(t *testing.T) { + // No global and no per-source → unlimited (0) + crl := newCustomRateLimit(nil, nil) + + rl, _ := resolveSourceRateLimit(0, crl, "crtsh") + assert.Equal(t, uint(0), rl) +} + +func TestResolveSourceRateLimit_ZeroValueRateLimit(t *testing.T) { + // Zero-value CustomRateLimit (nil maps) should not panic and fall back to global + rl, dur := resolveSourceRateLimit(5, &subscraping.CustomRateLimit{}, "crtsh") + assert.Equal(t, uint(5), rl) + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_PerSourceDefaultDuration(t *testing.T) { + // Per-source limit without explicit duration defaults to per-second + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 3}, + nil, + ) + + rl, dur := resolveSourceRateLimit(0, crl, "hackertarget") + assert.Equal(t, uint(3), rl) + assert.Equal(t, time.Second, dur) +} + +func TestBuildMultiRateLimiter_GlobalAppliedToAll(t *testing.T) { + // With -rl 5 and no per-source overrides, all sources should be rate-limited + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 5, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_NilRateLimit(t *testing.T) { + // nil rateLimit should not panic (guard in buildMultiRateLimiter) + agent := New([]string{"hackertarget"}, []string{}, false, false) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, nil) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_PerSourceWithDuration(t *testing.T) { + // Per-source limit with custom duration (e.g. -rls hackertarget=2/m) + agent := New([]string{"hackertarget", "crtsh"}, []string{}, false, false) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 2}, + map[string]time.Duration{"hackertarget": time.Minute}, + ) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestBuildMultiRateLimiter_UnlimitedWhenNoLimits(t *testing.T) { + // Without any rate limits, sources should get MaxUint32 (effectively unlimited) + agent := New([]string{"hackertarget"}, []string{}, false, false) + crl := newCustomRateLimit(nil, nil) + + limiter, err := agent.buildMultiRateLimiter(context.Background(), 0, crl) + require.NoError(t, err) + require.NotNil(t, limiter) +} + +func TestResolveSourceRateLimit_PerSourceExceedsGlobal(t *testing.T) { + // Per-source limit higher than global should be honoured + // (the source was explicitly allowed more than the default) + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 20}, + nil, + ) + + rl, dur := resolveSourceRateLimit(10, crl, "hackertarget") + assert.Equal(t, uint(20), rl, "per-source limit should take precedence even when > global") + assert.Equal(t, time.Second, dur) +} + +func TestResolveSourceRateLimit_CaseInsensitive(t *testing.T) { + // Source names should be case-insensitive for lookup + crl := newCustomRateLimit( + map[string]uint{"hackertarget": 5}, + nil, + ) + + // Try mixed case — should still match the lower-cased key + rl, _ := resolveSourceRateLimit(0, crl, "HackerTarget") + assert.Equal(t, uint(5), rl, "source name lookup should be case-insensitive") +} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index b22a4444..45fd53f1 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -26,20 +26,23 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" @@ -47,11 +50,14 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/submd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -61,6 +67,7 @@ var AllSources = [...]subscraping.Source{ &anubis.Source{}, &bevigil.Source{}, &bufferover.Source{}, + &builtwith.Source{}, &c99.Source{}, &censys.Source{}, &certspotter.Source{}, @@ -68,22 +75,29 @@ var AllSources = [...]subscraping.Source{ &chinaz.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, + &digitalyama.Source{}, &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, &dnsrepo.Source{}, + &domainsproject.Source{}, &driftnet.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, - &hunter.Source{}, + &hudsonrock.Source{}, &intelx.Source{}, - &netlas.Source{}, &leakix.Source{}, - &quake.Source{}, + &merklemap.Source{}, + &netlas.Source{}, + &onyphe.Source{}, + &profundis.Source{}, &pugrecon.Source{}, + &quake.Source{}, &rapiddns.Source{}, + // &reconcloud.Source{}, // failing due to cloudflare bot protection + &reconeer.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, @@ -91,18 +105,16 @@ var AllSources = [...]subscraping.Source{ &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, + &thc.Source{}, &threatbook.Source{}, &threatcrowd.Source{}, + // &threatminer.Source{}, // failing api + &urlscan.Source{}, &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, + &windvane.Source{}, &zoomeyeapi.Source{}, - &facebook.Source{}, - // &threatminer.Source{}, // failing api - // &reconcloud.Source{}, // failing due to cloudflare bot protection - &builtwith.Source{}, - &hudsonrock.Source{}, - &digitalyama.Source{}, &submd.Source{}, } @@ -174,9 +186,9 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } - // TODO: Consider refactoring this to avoid potential duplication issues for _, source := range sources { - if source.NeedsKey() { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { source.AddApiKeys([]string{apiKey}) } diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index dd00fd34..bd44d408 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -26,6 +26,7 @@ var ( "dnsdumpster", "dnsdb", "dnsrepo", + "domainsproject", "driftnet", "fofa", "fullhunt", @@ -33,6 +34,7 @@ var ( "hackertarget", "intelx", "netlas", + "onyphe", "quake", "pugrecon", "rapiddns", @@ -41,6 +43,7 @@ var ( "robtex", "rsecloud", "securitytrails", + "profundis", "shodan", "sitedossier", "threatbook", @@ -48,15 +51,18 @@ var ( "virustotal", "waybackarchive", "whoisxmlapi", + "windvane", "zoomeyeapi", - "hunter", "leakix", - "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", "hudsonrock", "digitalyama", + "merklemap", + "thc", + "urlscan", "submd", } @@ -73,28 +79,33 @@ var ( "crtsh", "digitorus", "dnsdumpster", + "domainsproject", "dnsrepo", "driftnet", "fofa", "fullhunt", "hackertarget", "intelx", + "onyphe", "quake", "redhuntlabs", "robtex", // "riddler", // failing due to cloudfront protection "rsecloud", "securitytrails", + "profundis", "shodan", + "windvane", "virustotal", "whoisxmlapi", - "hunter", "leakix", - "facebook", // "threatminer", // "reconcloud", + "reconeer", "builtwith", "digitalyama", + "thc", + "urlscan", "submd", } @@ -110,7 +121,8 @@ var ( "securitytrails", "virustotal", "leakix", - "facebook", + "merklemap", + "urlscan", // "reconcloud", } ) diff --git a/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go index 036588cc..c3316d63 100644 --- a/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -33,6 +33,10 @@ func TestSourcesWithoutKeys(t *testing.T) { "digitorus", // failing with "Failed to retrieve certificate" "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" + "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" + "leakix", // now requires API key (returns 401) + "reconeer", // now requires API key (returns 401) + "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index d7e4d887..38055b89 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -25,18 +25,20 @@ type ResolutionPool struct { // HostEntry defines a host with the source type HostEntry struct { - Domain string - Host string - Source string + Domain string + Host string + Source string + WildcardCertificate bool } // Result contains the result for a host resolution type Result struct { - Type ResultType - Host string - IP string - Error error - Source string + Type ResultType + Host string + IP string + Error error + Source string + WildcardCertificate bool } // ResultType is the type of result found @@ -92,13 +94,13 @@ func (r *ResolutionPool) InitWildcards(domain string) error { func (r *ResolutionPool) resolveWorker() { for task := range r.Tasks { if !r.removeWildcard { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate} continue } hosts, err := r.DNSClient.Lookup(task.Host) if err != nil { - r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} + r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate} continue } @@ -116,7 +118,7 @@ func (r *ResolutionPool) resolveWorker() { } if !skip { - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate} } } r.wg.Done() diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go index a0d3a7d1..a96fa70d 100644 --- a/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.8.0` +const version = `v2.13.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/pkg/runner/config.go b/pkg/runner/config.go index 1751c905..b0c65205 100644 --- a/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" fileutil "github.com/projectdiscovery/utils/file" ) @@ -25,7 +26,8 @@ func createProviderConfigYAML(configFilePath string) error { sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { - if source.NeedsKey() { + keyReq := source.KeyRequirement() + if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { sourceName := strings.ToLower(source.Name()) sourcesRequiringApiKeysMap[sourceName] = []string{} } @@ -46,7 +48,7 @@ func UnmarshalFrom(file string) error { for _, source := range passive.AllSources { sourceName := strings.ToLower(source.Name()) apiKeys := sourceApiKeysMap[sourceName] - if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 { + if len(apiKeys) > 0 { gologger.Debug().Msgf("API key(s) found for %s.", sourceName) source.AddApiKeys(apiKeys) } diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go index f71a347d..bd8953e5 100644 --- a/pkg/runner/enumerate.go +++ b/pkg/runner/enumerate.go @@ -67,6 +67,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: subdomain := replacer.Replace(result.Value) + // check if this subdomain is actually a wildcard subdomain + // that may have furthur subdomains associated with it + isWildcard := strings.Contains(result.Value, "*."+subdomain) // Validate the subdomain found and remove wildcards from if !strings.HasSuffix(subdomain, "."+domain) { @@ -90,10 +93,20 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // send the subdomain for resolution. if _, ok := uniqueMap[subdomain]; ok { skippedCounts[result.Source]++ + // even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard + // then we should mark it as wildcard + if !uniqueMap[subdomain].WildcardCertificate && isWildcard { + val := uniqueMap[subdomain] + val.WildcardCertificate = true + uniqueMap[subdomain] = val + } continue } - hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source} + hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard} + if r.options.ResultCallback != nil && !r.options.RemoveWildcard { + r.options.ResultCallback(&hostEntry) + } uniqueMap[subdomain] = hostEntry // If the user asked to remove wildcard then send on the resolve @@ -109,6 +122,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string if r.options.RemoveWildcard { close(resolutionPool.Tasks) } + wg.Done() }() @@ -125,9 +139,22 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Add the found subdomain to a map. if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result + if r.options.ResultCallback != nil { + r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate}) + } } } } + + // Merge wildcard certificate information from uniqueMap into foundResults + // This handles cases where a later source marked a subdomain as wildcard + // after it was already sent to the resolution pool + for host, result := range foundResults { + if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate { + result.WildcardCertificate = true + foundResults[host] = result + } + } } wg.Wait() outputWriter := NewOutputWriter(r.options.JSON) @@ -162,17 +189,6 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string numberOfSubDomains = len(uniqueMap) } - if r.options.ResultCallback != nil { - if r.options.RemoveWildcard { - for host, result := range foundResults { - r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source}) - } - } else { - for _, v := range uniqueMap { - r.options.ResultCallback(&v) - } - } - } gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration) if r.options.Statistics { diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 7f39e031..946c3027 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -15,6 +15,8 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" @@ -23,8 +25,8 @@ import ( var ( configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") - defaultConfigLocation = filepath.Join(configDir, "config.yaml") - defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml") + defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml")) + defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml")) ) // Options contains the configuration options for tuning @@ -222,16 +224,20 @@ func (options *Options) loadProvidersFrom(location string) { func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) - gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n") + gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n") gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig) for _, source := range passive.AllSources { - message := "%s\n" sourceName := source.Name() - if source.NeedsKey() { - message = "%s *\n" + switch source.KeyRequirement() { + case subscraping.RequiredKey: + gologger.Silent().Msgf("%s *\n", sourceName) + case subscraping.OptionalKey: + gologger.Silent().Msgf("%s ~\n", sourceName) + default: + gologger.Silent().Msgf("%s\n", sourceName) } - gologger.Silent().Msgf(message, sourceName) } } @@ -259,4 +265,5 @@ var defaultRateLimits = []string{ // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", + "urlscan=1/s", } diff --git a/pkg/runner/outputter.go b/pkg/runner/outputter.go index cfa4fc1a..2a41f512 100644 --- a/pkg/runner/outputter.go +++ b/pkg/runner/outputter.go @@ -19,22 +19,25 @@ type OutputWriter struct { } type jsonSourceResult struct { - Host string `json:"host"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourceIPResult struct { - Host string `json:"host"` - IP string `json:"ip"` - Input string `json:"input"` - Source string `json:"source"` + Host string `json:"host"` + IP string `json:"ip"` + Input string `json:"input"` + Source string `json:"source"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourcesResult struct { - Host string `json:"host"` - Input string `json:"input"` - Sources []string `json:"sources"` + Host string `json:"host"` + Input string `json:"input"` + Sources []string `json:"sources"` + WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } // NewOutputWriter creates a new OutputWriter @@ -117,7 +120,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. data.IP = result.IP data.Input = input data.Source = result.Source - + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(&data) if err != nil { return err @@ -130,7 +133,7 @@ func writeJSONHostIP(input string, results map[string]resolve.Result, writer io. func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error { hosts := make(map[string]resolve.HostEntry) for host, result := range results { - hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source} + hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate} } return o.WriteHost(input, hosts, writer) @@ -175,6 +178,7 @@ func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io data.Host = result.Host data.Input = input data.Source = result.Source + data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(data) if err != nil { return err @@ -223,11 +227,12 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr for host, sources := range sourceMap { sb.WriteString(host) sb.WriteString(",[") - sourcesString := "" + var sourcesString strings.Builder for source := range sources { - sourcesString += source + "," + sourcesString.WriteString(source) + sourcesString.WriteRune(',') } - sb.WriteString(strings.Trim(sourcesString, ", ")) + sb.WriteString(strings.TrimSuffix(sourcesString.String(), ",")) sb.WriteString("]\n") _, err := bufwriter.WriteString(sb.String()) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index f00ce641..b9dd844d 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -10,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "time" "github.com/projectdiscovery/gologger" contextutil "github.com/projectdiscovery/utils/context" @@ -61,11 +62,17 @@ func NewRunner(options *Options) (*Runner, error) { Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, + CustomDuration: mapsutil.SyncLockMap[string, time.Duration]{ + Map: make(map[string]time.Duration), + }, } for source, sourceRateLimit := range options.RateLimits.AsMap() { if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint { _ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount) + if sourceRateLimit.Duration > 0 { + _ = runner.rateLimit.CustomDuration.Set(source, sourceRateLimit.Duration) + } } } diff --git a/pkg/runner/stats.go b/pkg/runner/stats.go index d34b20f8..ccaa13e3 100644 --- a/pkg/runner/stats.go +++ b/pkg/runner/stats.go @@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) { if sourceStats.Skipped { skipped = append(skipped, fmt.Sprintf(" %s", source)) } else { - lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors)) + lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors)) } } if len(lines) > 0 { - gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) + gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68)) gologger.Print().Msg(strings.Join(lines, "\n")) gologger.Print().Msgf("\n") } diff --git a/pkg/subscraping/agent.go b/pkg/subscraping/agent.go index 4f2cab8c..6217ce58 100644 --- a/pkg/subscraping/agent.go +++ b/pkg/subscraping/agent.go @@ -46,7 +46,7 @@ func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLi Timeout: time.Duration(timeout) * time.Second, } - session := &Session{Client: client} + session := &Session{Client: client, Timeout: timeout} // Initiate rate limit instance session.MultiRateLimiter = multiRateLimiter diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go index 27e1f57b..c383e735 100644 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -23,6 +23,9 @@ type Source struct { timeTaken time.Duration results int errors int + requests int + apiKeys []string + skipped bool } // Run function returns all subdomains found with the service @@ -30,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -37,7 +41,15 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + s.requests++ + resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", + map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -64,8 +76,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range response.PassiveDNS { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } }() @@ -85,18 +101,24 @@ func (s *Source) HasRecursiveSupport() bool { return true } -func (s *Source) NeedsKey() bool { - return false +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } diff --git a/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go index 6efbd534..fa951457 100644 --- a/pkg/subscraping/sources/anubis/anubis.go +++ b/pkg/subscraping/sources/anubis/anubis.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -56,8 +59,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() @@ -78,8 +85,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -90,6 +101,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go index 2021e4c6..22197c7f 100644 --- a/pkg/subscraping/sources/bevigil/bevigil.go +++ b/pkg/subscraping/sources/bevigil/bevigil.go @@ -21,6 +21,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) + s.requests++ resp, err := session.Get(ctx, getUrl, "", map[string]string{ "X-Access-Token": randomApiKey, "User-Agent": "subfinder", }) @@ -70,8 +73,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() @@ -90,8 +97,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -102,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 207cb819..3c726607 100644 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { + s.requests++ resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) if err != nil && resp == nil { @@ -96,8 +99,12 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s for _, subdomain := range subdomains { for _, value := range session.Extractor.Extract(subdomain) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -115,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -127,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go index 4c6b516f..c500e6dc 100644 --- a/pkg/subscraping/sources/builtwith/builtwith.go +++ b/pkg/subscraping/sources/builtwith/builtwith.go @@ -35,6 +35,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -74,8 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}: + s.results++ + } } } }() @@ -96,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -108,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go index c2e45e81..54f8534c 100644 --- a/pkg/subscraping/sources/c99/c99.go +++ b/pkg/subscraping/sources/c99/c99.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -81,8 +84,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, data := range response.Subdomains { if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}: + s.results++ + } } } }() @@ -103,8 +110,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -115,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 4427821a..e913c864 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -2,55 +2,71 @@ package censys import ( + "bytes" "context" - "strconv" + "net/http" + "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - urlutil "github.com/projectdiscovery/utils/url" ) const ( + // maxCensysPages is the maximum number of pages to fetch from the API maxCensysPages = 10 - maxPerPage = 100 + // maxPerPage is the maximum number of results per page + maxPerPage = 100 + // baseURL is the Censys Platform API base URL + baseURL = "https://api.platform.censys.io" + // searchEndpoint is the global data search query endpoint + searchEndpoint = "/v3/global/search/query" + // queryPrefix is the Censys query language prefix for certificate name search + queryPrefix = "cert.names: " + // authHeaderPrefix is the Bearer token prefix for Authorization header + authHeaderPrefix = "Bearer " + // contentTypeJSON is the Content-Type header value for JSON + contentTypeJSON = "application/json" + // orgIDHeader is the header name for organization ID + orgIDHeader = "X-Organization-ID" ) +// apiKey holds the Personal Access Token and optional Organization ID +type apiKey struct { + pat string + orgID string +} + +// Platform API request body +type searchRequest struct { + Query string `json:"query"` + Fields []string `json:"fields,omitempty"` + PageSize int `json:"page_size,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + +// Platform API response structures type response struct { - Code int `json:"code"` - Status string `json:"status"` Result result `json:"result"` } type result struct { - Query string `json:"query"` - Total float64 `json:"total"` - DurationMS int `json:"duration_ms"` - Hits []hit `json:"hits"` - Links links `json:"links"` + Hits []hit `json:"hits"` + TotalHits int64 `json:"total_hits"` + NextPageToken string `json:"next_page_token"` } type hit struct { - Parsed parsed `json:"parsed"` - Names []string `json:"names"` - FingerprintSha256 string `json:"fingerprint_sha256"` + CertificateV1 certificateV1 `json:"certificate_v1"` } -type parsed struct { - ValidityPeriod validityPeriod `json:"validity_period"` - SubjectDN string `json:"subject_dn"` - IssuerDN string `json:"issuer_dn"` +type certificateV1 struct { + Resource resource `json:"resource"` } -type validityPeriod struct { - NotAfter string `json:"not_after"` - NotBefore string `json:"not_before"` -} - -type links struct { - Next string `json:"next"` - Prev string `json:"prev"` +type resource struct { + Names []string `json:"names"` } // Source is the passive scraping agent @@ -59,19 +75,16 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } -type apiKey struct { - token string - secret string -} - // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -79,37 +92,61 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + // PickRandom selects a random API key from configured keys. + // This enables load balancing when users configure multiple PATs + // (e.g., CENSYS_API_KEY=pat1:org1,pat2:org2) to distribute requests + // and avoid hitting rate limits on a single key. randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey.token == "" || randomApiKey.secret == "" { + if randomApiKey.pat == "" { s.skipped = true return } - certSearchEndpoint := "https://search.censys.io/api/v2/certificates/search" + apiURL := baseURL + searchEndpoint cursor := "" currentPage := 1 + for { - certSearchEndpointUrl, err := urlutil.Parse(certSearchEndpoint) + select { + case <-ctx.Done(): + return + default: + } + + reqBody := searchRequest{ + Query: queryPrefix + domain, + Fields: []string{"cert.names"}, + PageSize: maxPerPage, + } + if cursor != "" { + reqBody.Cursor = cursor + } + + bodyBytes, err := jsoniter.Marshal(reqBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } - certSearchEndpointUrl.Params.Add("q", domain) - certSearchEndpointUrl.Params.Add("per_page", strconv.Itoa(maxPerPage)) - if cursor != "" { - certSearchEndpointUrl.Params.Add("cursor", cursor) + headers := map[string]string{ + "Content-Type": contentTypeJSON, + "Authorization": authHeaderPrefix + randomApiKey.pat, + } + // Add Organization ID header if provided + if randomApiKey.orgID != "" { + headers[orgIDHeader] = randomApiKey.orgID } + s.requests++ resp, err := session.HTTPRequest( ctx, - "GET", - certSearchEndpointUrl.String(), + http.MethodPost, + apiURL, "", - nil, - nil, - subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret}, + headers, + bytes.NewReader(bodyBytes), + subscraping.BasicAuth{}, ) if err != nil { @@ -121,24 +158,25 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var censysResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) + _ = resp.Body.Close() if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - session.DiscardHTTPResponse(resp) return } - session.DiscardHTTPResponse(resp) - for _, hit := range censysResponse.Result.Hits { - for _, name := range hit.Names { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name} - s.results++ + for _, name := range hit.CertificateV1.Resource.Names { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}: + s.results++ + } } } - // Exit the censys enumeration if last page is reached - cursor = censysResponse.Result.Links.Next + cursor = censysResponse.Result.NextPageToken if cursor == "" || currentPage >= maxCensysPages { break } @@ -162,20 +200,34 @@ func (s *Source) HasRecursiveSupport() bool { return false } -func (s *Source) NeedsKey() bool { - return true +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey } +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +// AddApiKeys parses and adds API keys. +// Format: "PAT:ORG_ID" where ORG_ID is required for paid accounts. +// Example: "censys_xxx_token:12345678-91011-1213" func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - return apiKey{k, v} + s.apiKeys = subscraping.CreateApiKeys(keys, func(pat, orgID string) apiKey { + return apiKey{pat: pat, orgID: orgID} }) + // Also support single PAT without org ID for free users + for _, key := range keys { + if !strings.Contains(key, ":") && key != "" { + s.apiKeys = append(s.apiKeys, apiKey{pat: key, orgID: ""}) + } + } } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/censys/censys_test.go b/pkg/subscraping/sources/censys/censys_test.go new file mode 100644 index 00000000..f928698f --- /dev/null +++ b/pkg/subscraping/sources/censys/censys_test.go @@ -0,0 +1,137 @@ +package censys + +import ( + "context" + "math" + "net/http" + "testing" + "time" + + "github.com/projectdiscovery/ratelimit" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestMultiRateLimiter creates a MultiLimiter for testing +func createTestMultiRateLimiter(ctx context.Context) *ratelimit.MultiLimiter { + mrl, _ := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: "censys", + IsUnlimited: false, + MaxCount: math.MaxInt32, + Duration: time.Millisecond, + }) + return mrl +} + +func TestCensysSource_NoApiKey(t *testing.T) { + source := &Source{} + // Don't add any API keys + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: http.DefaultClient, + MultiRateLimiter: multiRateLimiter, + } + + ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "censys") + results := source.Run(ctxWithValue, "example.com", session) + + // Collect all results + var resultCount int + for range results { + resultCount++ + } + + // Should be skipped when no API key + stats := source.Statistics() + assert.True(t, stats.Skipped, "expected source to be skipped without API key") + assert.Equal(t, 0, resultCount, "expected no results when skipped") +} + +func TestCensysSource_ContextCancellation(t *testing.T) { + source := &Source{} + // Add a key with PAT:ORG_ID format + source.AddApiKeys([]string{"test_pat:test_org_id"}) + + ctx := context.Background() + multiRateLimiter := createTestMultiRateLimiter(ctx) + session := &subscraping.Session{ + Client: http.DefaultClient, + MultiRateLimiter: multiRateLimiter, + } + + // Create a context that will be cancelled + ctxCancellable, cancel := context.WithCancel(ctx) + ctxWithValue := context.WithValue(ctxCancellable, subscraping.CtxSourceArg, "censys") + + results := source.Run(ctxWithValue, "example.com", session) + + // Cancel immediately + cancel() + + // Should exit quickly without blocking + done := make(chan struct{}) + go func() { + for range results { + // drain + } + close(done) + }() + + select { + case <-done: + // Good - completed quickly + case <-time.After(2 * time.Second): + t.Fatal("context cancellation did not stop the source in time") + } +} + +func TestCensysSource_Metadata(t *testing.T) { + source := &Source{} + + assert.Equal(t, "censys", source.Name()) + assert.True(t, source.IsDefault()) + assert.False(t, source.HasRecursiveSupport()) + assert.True(t, source.NeedsKey()) +} + +func TestCensysSource_AddApiKeys(t *testing.T) { + t.Run("PAT with OrgID", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_1:org_id_1", "pat_token_2:org_id_2"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 2) + assert.Equal(t, "pat_token_1", source.apiKeys[0].pat) + assert.Equal(t, "org_id_1", source.apiKeys[0].orgID) + assert.Equal(t, "pat_token_2", source.apiKeys[1].pat) + assert.Equal(t, "org_id_2", source.apiKeys[1].orgID) + }) + + t.Run("PAT without OrgID (free user)", func(t *testing.T) { + source := &Source{} + keys := []string{"pat_token_only"} + source.AddApiKeys(keys) + + require.Len(t, source.apiKeys, 1) + assert.Equal(t, "pat_token_only", source.apiKeys[0].pat) + assert.Equal(t, "", source.apiKeys[0].orgID) + }) +} + +func TestCensysSource_Statistics(t *testing.T) { + source := &Source{ + errors: 2, + results: 10, + timeTaken: 5 * time.Second, + skipped: false, + } + + stats := source.Statistics() + assert.Equal(t, 2, stats.Errors) + assert.Equal(t, 10, stats.Results) + assert.Equal(t, 5*time.Second, stats.TimeTaken) + assert.False(t, stats.Skipped) +} diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index e501e173..86e0160e 100644 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Authorization": "Bearer " + randomApiKey} cookies := "" + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -66,20 +69,29 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } - // if the number of responses is zero, close the channel and return. if len(response) == 0 { return } id := response[len(response)-1].ID for { + select { + case <-ctx.Done(): + return + default: + } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) + s.requests++ resp, err := session.Get(ctx, reqURL, cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -103,8 +115,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, cert := range response { for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -128,8 +144,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -140,6 +160,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go index 2cc87367..13824d91 100644 --- a/pkg/subscraping/sources/chaos/chaos.go +++ b/pkg/subscraping/sources/chaos/chaos.go @@ -16,14 +16,16 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } // Run function returns all subdomains found with the service -func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { +func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,9 +40,15 @@ func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) < } chaosClient := chaos.New(randomApiKey) + s.requests++ for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { + select { + case <-ctx.Done(): + return + default: + } if result.Error != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} s.errors++ @@ -69,8 +77,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -81,6 +93,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go index c6e43c05..7e2045aa 100644 --- a/pkg/subscraping/sources/chinaz/chinaz.go +++ b/pkg/subscraping/sources/chinaz/chinaz.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -55,9 +58,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if SubdomainList.ToBool() { _data := []byte(SubdomainList.ToString()) for i := 0; i < SubdomainList.Size(); i++ { + select { + case <-ctx.Done(): + return + default: + } subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } else { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -82,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -94,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index 09ad4967..9dd4d425 100644 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, indexURL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -105,8 +108,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -117,6 +124,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } @@ -128,6 +136,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se return false default: var headers = map[string]string{"Host": "index.commoncrawl.org"} + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -138,6 +147,12 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + default: + } line := scanner.Text() if line == "" { continue @@ -145,13 +160,17 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { if subdomain != "" { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return false + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index 76ac02b4..a22b5bdc 100644 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,7 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { - db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") + // connect_timeout: limits connection establishment time (in seconds) + // statement_timeout: limits query execution time (in milliseconds) + connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000) + db, err := sql.Open("postgres", connStr) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -114,8 +119,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio var count int var data string - // Parse all the rows getting subdomains for rows.Next() { + select { + case <-ctx.Done(): + return count + default: + } err := rows.Scan(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -127,8 +136,12 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return count + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -137,6 +150,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -157,11 +171,20 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { + select { + case <-ctx.Done(): + return true + default: + } for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + select { + case <-ctx.Done(): + return true + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: + s.results++ + } } } } @@ -183,8 +206,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -195,6 +222,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go index 7ab71957..5beae821 100644 --- a/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -34,6 +35,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -48,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + s.requests++ resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -94,8 +97,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() @@ -115,8 +122,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -127,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go index 8c680a2b..226469a2 100644 --- a/pkg/subscraping/sources/digitorus/digitorus.go +++ b/pkg/subscraping/sources/digitorus/digitorus.go @@ -18,6 +18,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -25,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -32,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { @@ -50,16 +53,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } subdomains := session.Extractor.Extract(line) for _, subdomain := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, ".")}: + s.results++ } - s.results++ } } }() @@ -80,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -92,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go index e5dc0e24..ce299c1c 100644 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results uint64 + requests int skipped bool } @@ -51,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -70,6 +72,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se "Accept": "application/x-ndjson", } + s.requests++ offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -85,8 +88,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se queryParams.Add("swclient", "subfinder") for { + select { + case <-ctx.Done(): + return + default: + } url := urlTemplate + queryParams.Encode() + s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} @@ -98,6 +107,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var respCond string reader := bufio.NewReader(resp.Body) for { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } n, err := reader.ReadBytes('\n') if err == io.EOF { break @@ -117,21 +132,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Condition is a scalar enum of string values: {“begin”, “ongoing”, “succeeded”, “limited”, “failed”}. - // "begin" will be the initiating Condition, this can be safely ignored. The data of interest will be in - // objects with Condition "" or "ongoing". Conditions "succeeded", "limited", and "failed" are terminating conditions. - // See https://www.domaintools.com/resources/user-guides/farsight-streaming-api-framing-protocol-documentation/ - // for more details respCond = response.Condition if respCond == "" || respCond == "ongoing" { if response.Obj.Name != "" { - results <- subscraping.Result{ - Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, "."), + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, ".")}: + s.results++ } - s.results++ } } else if respCond != "begin" { - // if the respCond is not "", "ongoing", or "begin", then it is a terminating condition, so break out of the loop break } } @@ -175,8 +187,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -187,6 +203,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: int(s.results), + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 72be41c3..fec6f73a 100644 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -25,6 +25,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -65,8 +68,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, record := range append(response.A, response.Ns...) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}: + s.results++ + } } }() @@ -87,8 +94,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -99,6 +110,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go index e98c8239..14e1084c 100644 --- a/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -28,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se token := randomApiInfo[0] apiKey := randomApiInfo[1] + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -74,10 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, sub := range result { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, "."), + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}: + s.results++ } - s.results++ } }() @@ -98,8 +103,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -110,6 +119,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/domainsproject/domainsproject.go b/pkg/subscraping/sources/domainsproject/domainsproject.go new file mode 100644 index 00000000..b6c7eab8 --- /dev/null +++ b/pkg/subscraping/sources/domainsproject/domainsproject.go @@ -0,0 +1,145 @@ +// Package domainsproject logic +package domainsproject + +import ( + "context" + "fmt" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +type apiKey struct { + username string + password string +} + +type domainsProjectResponse struct { + Domains []string `json:"domains"` + Error string `json:"error"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.username == "" || randomApiKey.password == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) + s.requests++ + resp, err := session.HTTPRequest( + ctx, + "GET", + searchURL, + "", + nil, + nil, + subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, + ) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + var response domainsProjectResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if response.Error != "" { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), + } + s.errors++ + return + } + + for _, subdomain := range response.Domains { + if !strings.HasPrefix(subdomain, ".") { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "domainsproject" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go index 758bbfa4..194952b8 100644 --- a/pkg/subscraping/sources/driftnet/driftnet.go +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors atomic.Int32 results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -65,6 +66,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) + s.requests.Store(0) // Waitgroup for subsources var wg sync.WaitGroup @@ -104,9 +106,14 @@ func (s *Source) HasRecursiveSupport() bool { return true } +// KeyRequirement indicates that we need an API key +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + // NeedsKey indicates that we need an API key func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys provides us with the API key(s) @@ -119,6 +126,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } @@ -139,6 +147,7 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc // Request requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + s.requests.Add(1) resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { // HTTP 204 is not an error from the Driftnet API @@ -176,17 +185,24 @@ func (s *Source) runSubsource(ctx context.Context, domain string, session *subsc } for subdomain := range summary.Summary.Values { - // We can get certificate results which aren't actually subdomains of the target domain. Skip them. + select { + case <-ctx.Done(): + wg.Done() + return + default: + } if !strings.HasSuffix(subdomain, "."+domain) { continue } - // Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints) if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain, + select { + case <-ctx.Done(): + wg.Done() + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) } - s.results.Add(1) } } diff --git a/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go deleted file mode 100644 index f9643600..00000000 --- a/pkg/subscraping/sources/facebook/ctlogs.go +++ /dev/null @@ -1,200 +0,0 @@ -package facebook - -import ( - "context" - "encoding/json" - "fmt" - "io" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - errorutil "github.com/projectdiscovery/utils/errors" - "github.com/projectdiscovery/utils/generic" - urlutil "github.com/projectdiscovery/utils/url" -) - -// source: https://developers.facebook.com/tools/ct -// api-docs: https://developers.facebook.com/docs/certificate-transparency-api -// ratelimit: ~20,000 req/hour per appID https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ - -var ( - domainsPerPage = "1000" - authUrl = "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials" - domainsUrl = "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=%s&limit=" + domainsPerPage -) - -type apiKey struct { - AppID string - Secret string - AccessToken string // obtained by calling - // https://graph.facebook.com/oauth/access_token?client_id=APP_ID&client_secret=APP_SECRET&grant_type=client_credentials - Error error // error while fetching access token -} - -// FetchAccessToken fetches the access token for the api key -// using app id and secret -func (k *apiKey) FetchAccessToken() { - if generic.EqualsAny("", k.AppID, k.Secret) { - k.Error = fmt.Errorf("invalid app id or secret") - return - } - resp, err := retryablehttp.Get(fmt.Sprintf(authUrl, k.AppID, k.Secret)) - if err != nil { - k.Error = err - return - } - defer func() { - if err := resp.Body.Close(); err != nil { - gologger.Error().Msgf("error closing response body: %s", err) - } - }() - bin, err := io.ReadAll(resp.Body) - if err != nil { - k.Error = err - return - } - auth := &authResponse{} - if err := json.Unmarshal(bin, auth); err != nil { - k.Error = err - return - } - if auth.AccessToken == "" { - k.Error = fmt.Errorf("invalid response from facebook got %v", string(bin)) - return - } - k.AccessToken = auth.AccessToken -} - -// IsValid returns true if the api key is valid -func (k *apiKey) IsValid() bool { - return k.AccessToken != "" -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - if len(s.apiKeys) == 0 { - s.skipped = true - close(results) - return results - } - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - key := subscraping.PickRandom(s.apiKeys, s.Name()) - domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain) - - for { - // unfortunately, this cannot be parllelized since pagination is cursor based - resp, err := session.Get(ctx, domainsURL, "", nil) - if err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - bin, err := io.ReadAll(resp.Body) - if err != nil { - s.errors++ - gologger.Verbose().Msgf("failed to read response body: %s\n", err) - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - session.DiscardHTTPResponse(resp) - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - s.errors++ - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errorutil.NewWithErr(err).Msgf("failed to unmarshal response: %s", string(bin))} - return - } - for _, v := range response.Data { - for _, domain := range v.Domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} - s.results++ - } - } - if response.Paging.Next == "" { - break - } - // cursor includes api key so no need to update it - domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage) - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "facebook" -} - -// IsDefault returns true if the source should be queried by default -func (s *Source) IsDefault() bool { - return true -} - -// accepts subdomains (e.g. subdomain.domain.tld) -// but also returns all SANs for a certificate which may not match the domain -func (s *Source) HasRecursiveSupport() bool { - return true -} - -// NeedsKey returns true if the source requires an API key -func (s *Source) NeedsKey() bool { - return true -} - -// AddApiKeys adds api keys to the source -func (s *Source) AddApiKeys(keys []string) { - allapikeys := subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - apiKey := apiKey{AppID: k, Secret: v} - apiKey.FetchAccessToken() - if apiKey.Error != nil { - gologger.Warning().Msgf("Could not fetch access token for %s: %s\n", k, apiKey.Error) - } - return apiKey - }) - // filter out invalid keys - for _, key := range allapikeys { - if key.IsValid() { - s.apiKeys = append(s.apiKeys, key) - } - } -} - -// Statistics returns the statistics for the source -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func updateParamInURL(url, param, value string) string { - urlx, err := urlutil.Parse(url) - if err != nil { - return url - } - urlx.Params.Set(param, value) - return urlx.String() -} diff --git a/pkg/subscraping/sources/facebook/ctlogs_test.go b/pkg/subscraping/sources/facebook/ctlogs_test.go deleted file mode 100644 index e9204055..00000000 --- a/pkg/subscraping/sources/facebook/ctlogs_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package facebook - -import ( - "encoding/json" - "fmt" - "io" - "os" - "strings" - "testing" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/utils/generic" -) - -var ( - fb_API_ID = "$FB_APP_ID" - fb_API_SECRET = "$FB_APP_SECRET" -) - -func TestFacebookSource(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - updateWithEnv(&fb_API_ID) - updateWithEnv(&fb_API_SECRET) - if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) { - t.SkipNow() - } - k := apiKey{ - AppID: fb_API_ID, - Secret: fb_API_SECRET, - } - k.FetchAccessToken() - if k.Error != nil { - t.Fatal(k.Error) - } - - fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken) - resp, err := retryablehttp.Get(fetchURL) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := resp.Body.Close(); err != nil { - gologger.Error().Msgf("error closing response body: %s", err) - } - }() - bin, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - response := &response{} - if err := json.Unmarshal(bin, response); err != nil { - t.Fatal(err) - } - if len(response.Data) == 0 { - t.Fatal("no data found") - } -} - -func updateWithEnv(key *string) { - if key == nil { - return - } - value := *key - if strings.HasPrefix(value, "$") { - *key = os.Getenv(value[1:]) - } -} diff --git a/pkg/subscraping/sources/facebook/types.go b/pkg/subscraping/sources/facebook/types.go deleted file mode 100644 index 7a0fab23..00000000 --- a/pkg/subscraping/sources/facebook/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package facebook - -type authResponse struct { - AccessToken string `json:"access_token"` -} - -/* -{ - "data": [ - { - "domains": [ - "docs.hackerone.com" - ], - "id": "10056051421102939" - }, - ... - ], - "paging": { - "cursors": { - "before": "MTAwNTYwNTE0MjExMDI5MzkZD", - "after": "Njc0OTczNTA5NTA1MzUxNwZDZD" - }, - "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD" - } -} -*/ -// example response - -type response struct { - Data []struct { - Domains []string `json:"domains"` - } `json:"data"` - Paging struct { - Next string `json:"next"` - } `json:"paging"` -} diff --git a/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go index 78be69d1..48ad360b 100644 --- a/pkg/subscraping/sources/fofa/fofa.go +++ b/pkg/subscraping/sources/fofa/fofa.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // fofa api doc https://fofa.info/static_pages/api_help qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -83,6 +86,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Size > 0 { for _, subdomain := range response.Results { + select { + case <-ctx.Done(): + return + default: + } if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { subdomain = subdomain[strings.Index(subdomain, "//")+2:] } @@ -112,8 +120,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -126,6 +138,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go index 60c64e52..237ae286 100644 --- a/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -61,8 +64,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: + s.results++ + } } }() @@ -82,8 +89,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -94,6 +105,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go index 2e8b3224..e92d9fdf 100644 --- a/pkg/subscraping/sources/github/github.go +++ b/pkg/subscraping/sources/github/github.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -41,16 +42,18 @@ type response struct { type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -86,11 +89,12 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } // Initial request to GitHub search + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -111,7 +115,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -121,7 +125,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -129,11 +133,16 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * linksHeader := linkheader.Parse(resp.Header.Get("Link")) // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) @@ -147,11 +156,22 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r errChan := make(chan error, len(items)) for _, responseItem := range items { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } wg.Add(1) go func(responseItem item) { defer wg.Done() - // find subdomains in code + select { + case <-ctx.Done(): + return + default: + } + + s.requests.Add(1) resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { @@ -164,23 +184,42 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r if resp.StatusCode == http.StatusOK { scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) + } } } session.DiscardHTTPResponse(resp) } - // find subdomains in text matches for _, textMatch := range responseItem.TextMatches { + select { + case <-ctx.Done(): + return + default: + } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: + s.results.Add(1) + } } } }(responseItem) @@ -231,8 +270,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -241,8 +284,9 @@ func (s *Source) AddApiKeys(keys []string) { func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go index 902ef469..944f4005 100644 --- a/pkg/subscraping/sources/gitlab/gitlab.go +++ b/pkg/subscraping/sources/gitlab/gitlab.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "time" jsoniter "github.com/json-iterator/go" @@ -20,8 +21,9 @@ import ( type Source struct { apiKeys []string timeTaken time.Duration - errors int - results int + errors atomic.Int32 + results atomic.Int32 + requests atomic.Int32 skipped bool } @@ -35,8 +37,9 @@ type item struct { // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 + s.errors.Store(0) + s.results.Store(0) + s.requests.Store(0) go func() { defer func(startTime time.Time) { @@ -66,10 +69,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * default: } + s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) session.DiscardHTTPResponse(resp) return } @@ -80,7 +84,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * err = jsoniter.NewDecoder(resp.Body).Decode(&items) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -91,13 +95,14 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) + s.requests.Add(1) resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { session.DiscardHTTPResponse(resp) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } } @@ -111,7 +116,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } for _, subdomain := range domainRegexp.FindAllString(line, -1) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + s.results.Add(1) } } session.DiscardHTTPResponse(resp) @@ -120,15 +125,18 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * }(it) } - // Links header, first, next, last... linksHeader := linkheader.Parse(resp.Header.Get("Link")) - // Process the next link recursively for _, link := range linksHeader { + select { + case <-ctx.Done(): + return + default: + } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + s.errors.Add(1) return } @@ -157,8 +165,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -168,8 +180,9 @@ func (s *Source) AddApiKeys(keys []string) { // Statistics returns the statistics for the source func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index c8495772..2c00e018 100644 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -12,9 +12,12 @@ import ( // Source is the passive scraping agent type Source struct { + apiKeys []string timeTaken time.Duration errors int results int + requests int + skipped bool } // Run function returns all subdomains found with the service @@ -22,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,7 +33,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain)) + htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + } + + htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) + + s.requests++ + resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -46,14 +59,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } match := session.Extractor.Extract(line) for _, subdomain := range match { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -74,12 +96,16 @@ func (s *Source) HasRecursiveSupport() bool { return true } -func (s *Source) NeedsKey() bool { - return false +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { @@ -87,5 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go index 51d6bc2f..79eca069 100644 --- a/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -26,6 +26,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -60,8 +63,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { for _, subdomain := range session.Extractor.Extract(record.URL) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -83,8 +90,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -95,6 +106,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/hunter/hunter.go b/pkg/subscraping/sources/hunter/hunter.go deleted file mode 100644 index 96056327..00000000 --- a/pkg/subscraping/sources/hunter/hunter.go +++ /dev/null @@ -1,131 +0,0 @@ -package hunter - -import ( - "context" - "encoding/base64" - "fmt" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -type hunterResp struct { - Code int `json:"code"` - Data hunterData `json:"data"` - Message string `json:"message"` -} - -type infoArr struct { - URL string `json:"url"` - IP string `json:"ip"` - Port int `json:"port"` - Domain string `json:"domain"` - Protocol string `json:"protocol"` -} - -type hunterData struct { - InfoArr []infoArr `json:"arr"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var pages = 1 - for currentPage := 1; currentPage <= pages; currentPage++ { - // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 - qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage)) - if err != nil && resp == nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response hunterResp - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - session.DiscardHTTPResponse(resp) - - if response.Code == 401 || response.Code == 400 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), - } - s.errors++ - return - } - - if response.Data.Total > 0 { - for _, hunterInfo := range response.Data.InfoArr { - subdomain := hunterInfo.Domain - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - } - pages = int(response.Data.Total/1000) + 1 - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "hunter" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go index b3a3a45a..560f4fe6 100644 --- a/pkg/subscraping/sources/intelx/intelx.go +++ b/pkg/subscraping/sources/intelx/intelx.go @@ -43,6 +43,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -56,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -85,6 +87,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -107,6 +110,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 for status == 0 || status == 3 { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -134,10 +143,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se status = response.Status for _, hostname := range response.Selectors { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}: + s.results++ } - s.results++ } } }() @@ -158,8 +169,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -172,6 +187,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go index 4159e6ab..84c07b4d 100644 --- a/pkg/subscraping/sources/leakix/leakix.go +++ b/pkg/subscraping/sources/leakix/leakix.go @@ -16,6 +16,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -40,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers["api-key"] = randomApiKey } // Request + s.requests++ resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -64,10 +67,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ } - s.results++ } }() return results @@ -86,8 +91,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -98,6 +107,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/merklemap/merklemap.go b/pkg/subscraping/sources/merklemap/merklemap.go new file mode 100644 index 00000000..b27f689d --- /dev/null +++ b/pkg/subscraping/sources/merklemap/merklemap.go @@ -0,0 +1,174 @@ +// Package merklemap logic +package merklemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/url" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + // Pick an API key, skip if no key is found + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Default headers + headers := map[string]string{ + "accept": "application/json", + // Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api + "User-Agent": "subfinder", + "Authorization": "Bearer " + randomApiKey, + } + + // Fetch all pages with pagination + // https://www.merklemap.com/documentation/search + s.fetchAllPages(ctx, domain, headers, session, results) + }() + return results +} + +// fetchAllPages fetches all pages of results using pagination +func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + baseURL := "https://api.merklemap.com/v1/search?query=" + url.QueryEscape("*."+domain) + totalCount := math.MaxInt + processedResults := 0 + + // Iterate through all pages + for page := 0; processedResults < totalCount; page++ { + pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if page == 0 { + totalCount = pageResp.Count + } + + // Stop if this page returned no results + if len(pageResp.Results) == 0 { + break + } + + for _, result := range pageResp.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: result.Hostname, + } + s.results++ + processedResults++ + } + + } +} + +// fetchPage fetches a single page of results +func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { + url := baseURL + "&page=" + strconv.Itoa(page) + + s.requests++ + resp, err := session.Get(ctx, url, "", headers) + if err != nil { + return nil, err + } + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, err) + } + return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, string(respBody)) + } + + var pageResponse response + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(respBody)) + if err := decoder.Decode(&pageResponse); err != nil { + return nil, err + } + + return &pageResponse, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "merklemap" +} + +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} + +type response struct { + Count int `json:"count"` + Results []struct { + Hostname string `json:"hostname"` + SubjectCommonName string `json:"subject_common_name"` + FirstSeen string `json:"first_seen"` + } `json:"results"` +} diff --git a/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go index c29bc2c8..fbf65e55 100644 --- a/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -40,6 +40,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -47,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,7 +65,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ + s.requests++ + resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, }, nil, subscraping.BasicAuth{}) @@ -72,19 +75,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return - } else if resp.StatusCode != 200 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + } else if resp1.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)} s.errors++ return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp1.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp1.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ @@ -120,7 +123,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ + s.requests++ + resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) @@ -130,20 +134,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp2.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err = io.ReadAll(resp.Body) + body, err = io.ReadAll(resp2.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } - if resp.StatusCode == 429 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + if resp2.StatusCode == 429 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)} s.errors++ return } @@ -158,10 +162,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, item := range data { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain}: + s.results++ } - s.results++ } }() @@ -182,8 +188,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -194,6 +204,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/onyphe/onyphe.go b/pkg/subscraping/sources/onyphe/onyphe.go new file mode 100644 index 00000000..8c58dd45 --- /dev/null +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -0,0 +1,318 @@ +// Package onyphe logic +package onyphe + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type OnypheResponse struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Total int `json:"total"` + MaxPage int `json:"max_page"` +} + +type Result struct { + Subdomains []string `json:"subdomains"` + Hostname string `json:"hostname"` + Forward string `json:"forward"` + Reverse string `json:"reverse"` + Host string `json:"host"` + Domain string `json:"domain"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} + + page := 1 + pageSize := 1000 + + for { + select { + case <-ctx.Done(): + return + default: + } + var resp *http.Response + var err error + + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", + url.QueryEscape("category:resolver domain:"+domain), page, pageSize) + s.requests++ + resp, err = session.Get(ctx, urlWithQuery, "", headers) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var respOnyphe OnypheResponse + err = json.NewDecoder(resp.Body).Decode(&respOnyphe) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + session.DiscardHTTPResponse(resp) + + for _, record := range respOnyphe.Results { + select { + case <-ctx.Done(): + return + default: + } + for _, subdomain := range record.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + + if record.Hostname != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if record.Forward != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward} + s.results++ + } + + if record.Reverse != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse} + s.results++ + } + } + + if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage { + break + } + + page++ + + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "onyphe" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +type OnypheResponseRaw struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page json.RawMessage `json:"page"` + PageSize json.RawMessage `json:"page_size"` + Total json.RawMessage `json:"total"` + MaxPage json.RawMessage `json:"max_page"` +} + +func (o *OnypheResponse) UnmarshalJSON(data []byte) error { + var raw OnypheResponseRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + o.Error = raw.Error + o.Results = raw.Results + + if pageStr := string(raw.Page); pageStr != "" { + if page, err := strconv.Atoi(pageStr); err == nil { + o.Page = page + } else { + var pageStrQuoted string + if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil { + if page, err := strconv.Atoi(pageStrQuoted); err == nil { + o.Page = page + } + } + } + } + + if pageSizeStr := string(raw.PageSize); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { + o.PageSize = pageSize + } else { + var pageSizeStrQuoted string + if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil { + if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil { + o.PageSize = pageSize + } + } + } + } + + if totalStr := string(raw.Total); totalStr != "" { + if total, err := strconv.Atoi(totalStr); err == nil { + o.Total = total + } else { + var totalStrQuoted string + if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil { + if total, err := strconv.Atoi(totalStrQuoted); err == nil { + o.Total = total + } + } + } + } + + if maxPageStr := string(raw.MaxPage); maxPageStr != "" { + if maxPage, err := strconv.Atoi(maxPageStr); err == nil { + o.MaxPage = maxPage + } else { + var maxPageStrQuoted string + if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil { + if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil { + o.MaxPage = maxPage + } + } + } + } + + return nil +} + +type ResultRaw struct { + Subdomains json.RawMessage `json:"subdomains"` + Hostname json.RawMessage `json:"hostname"` + Forward json.RawMessage `json:"forward"` + Reverse json.RawMessage `json:"reverse"` + Host json.RawMessage `json:"host"` + Domain json.RawMessage `json:"domain"` +} + +func (r *Result) UnmarshalJSON(data []byte) error { + var raw ResultRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + var subdomains []string + if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil { + r.Subdomains = subdomains + } else { + var subdomainStr string + if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil { + r.Subdomains = []string{subdomainStr} + } + } + + if len(raw.Hostname) > 0 { + var hostnameStr string + if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil { + r.Hostname = hostnameStr + } else { + var hostnameArr []string + if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 { + r.Hostname = hostnameArr[0] + } + } + } + + if len(raw.Forward) > 0 { + _ = json.Unmarshal(raw.Forward, &r.Forward) + } + + if len(raw.Reverse) > 0 { + _ = json.Unmarshal(raw.Reverse, &r.Reverse) + } + + if len(raw.Host) > 0 { + var hostStr string + if err := json.Unmarshal(raw.Host, &hostStr); err == nil { + r.Host = hostStr + } else { + var hostArr []string + if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 { + r.Host = hostArr[0] + } + } + } + + if len(raw.Domain) > 0 { + var domainStr string + if err := json.Unmarshal(raw.Domain, &domainStr); err == nil { + r.Domain = domainStr + } else { + var domainArr []string + if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 { + r.Domain = domainArr[0] + } + } + } + + return nil +} diff --git a/pkg/subscraping/sources/profundis/profundis.go b/pkg/subscraping/sources/profundis/profundis.go new file mode 100644 index 00000000..09cd70ea --- /dev/null +++ b/pkg/subscraping/sources/profundis/profundis.go @@ -0,0 +1,135 @@ +// Package profundis logic +package profundis + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "strings" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + requestBody, err := json.Marshal(map[string]string{"domain": domain}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + headers := map[string]string{ + "Content-Type": "application/json", + "X-API-KEY": randomApiKey, + "Accept": "text/event-stream", + } + + s.requests++ + resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", + headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: line}: + s.results++ + } + } + + if err := scanner.Err(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() + + return results +} + +func (s *Source) Name() string { + return "profundis" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go index e7f2255f..8c0aae96 100644 --- a/pkg/subscraping/sources/pugrecon/pugrecon.go +++ b/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -32,6 +32,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -40,6 +41,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -71,6 +73,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } apiURL := "https://pugrecon.com/api/v1/domains" + s.requests++ resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -106,10 +109,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range response.Results { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name}: + s.results++ } - s.results++ } }() @@ -131,9 +136,14 @@ func (s *Source) HasRecursiveSupport() bool { return false } +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + // NeedsKey returns true as this source requires an API key. func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys adds the API keys for the source. @@ -148,5 +158,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go index 5cac4ed2..6ad6bee7 100644 --- a/pkg/subscraping/sources/quake/quake.go +++ b/pkg/subscraping/sources/quake/quake.go @@ -36,6 +36,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -44,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,7 +65,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var totalResults = -1 for { + select { + case <-ctx.Done(): + return + default: + } var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + s.requests++ resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) @@ -97,6 +105,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, quakeDomain := range response.Data { + select { + case <-ctx.Done(): + return + default: + } subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "暂无权限") { continue @@ -129,8 +142,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -143,5 +160,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go index 89ec7063..e2e679a4 100644 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -19,6 +19,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -26,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 maxPages := 1 for { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -56,8 +64,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } if maxPages == 1 { @@ -93,8 +105,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -106,5 +122,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go index 8aacc7d4..d572d525 100644 --- a/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -29,6 +29,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -63,8 +66,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}: + s.results++ + } } } }() @@ -85,8 +92,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -98,5 +109,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/reconeer/reconeer.go b/pkg/subscraping/sources/reconeer/reconeer.go new file mode 100644 index 00000000..d0e905be --- /dev/null +++ b/pkg/subscraping/sources/reconeer/reconeer.go @@ -0,0 +1,116 @@ +package reconeer + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Subdomains []subdomain `json:"subdomains"` +} + +type subdomain struct { + Subdomain string `json:"subdomain"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + headers := map[string]string{ + "Accept": "application/json", + } + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["X-API-KEY"] = randomApiKey + } + apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) + s.requests++ + resp, err := session.Get(ctx, apiURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + defer session.DiscardHTTPResponse(resp) + + if resp.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors++ + return + } + var responseData response + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&responseData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + for _, result := range responseData.Subdomains { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: + s.results++ + } + } + }() + return results +} + +func (s *Source) Name() string { + return "reconeer" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.OptionalKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index a07e9a5b..825161ee 100644 --- a/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -28,6 +28,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 pageSize := 1000 go func() { defer func(startTime time.Time) { @@ -56,6 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -76,7 +79,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { + select { + case <-ctx.Done(): + return + default: + } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) + s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} @@ -96,14 +105,22 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } } else { for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } @@ -123,8 +140,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -137,5 +158,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go index 9a3401c2..142fc6f3 100644 --- a/pkg/subscraping/sources/riddler/riddler.go +++ b/pkg/subscraping/sources/riddler/riddler.go @@ -15,6 +15,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -22,6 +23,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -29,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -39,13 +42,24 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + default: + } line := scanner.Text() if line == "" { continue } for _, subdomain := range session.Extractor.Extract(line) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + session.DiscardHTTPResponse(resp) + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } session.DiscardHTTPResponse(resp) @@ -67,8 +81,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -80,5 +98,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go index 9214363e..25f4e041 100644 --- a/pkg/subscraping/sources/robtex/robtext.go +++ b/pkg/subscraping/sources/robtex/robtext.go @@ -62,6 +62,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, result := range ips { + select { + case <-ctx.Done(): + return + default: + } if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) if err != nil { @@ -70,8 +75,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } for _, result := range domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}: + s.results++ + } } } } @@ -122,8 +131,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { diff --git a/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go index 8601b773..577ff0b3 100644 --- a/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -24,6 +24,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se fetchSubdomains := func(endpoint string) { page := 1 for { + select { + case <-ctx.Done(): + return + default: + } + s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,8 +76,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range rseCloudResponse.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } if page >= rseCloudResponse.TotalPages { @@ -99,8 +111,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -113,5 +129,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index e6d38fe4..c852b52f 100644 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -31,6 +31,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -39,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -56,18 +58,26 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} for { + select { + case <-ctx.Done(): + return + default: + } var resp *http.Response var err error if scrollId == "" { var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) + s.requests++ resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) } if err != nil && ptr.Safe(resp).StatusCode == 403 { + s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) } @@ -90,11 +100,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: + s.results++ + } } for _, subdomain := range securityTrailsResponse.Subdomains { + select { + case <-ctx.Done(): + return + default: + } if strings.HasSuffix(subdomain, ".") { subdomain += domain } else { @@ -128,8 +147,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -142,5 +165,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index d2224a81..f5f3d365 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -33,6 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -48,8 +50,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se page := 1 for { + select { + case <-ctx.Done(): + return + default: + } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) + s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) @@ -75,6 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, data := range response.Subdomains { + select { + case <-ctx.Done(): + return + default: + } value := fmt.Sprintf("%s.%s", data, response.Domain) results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, @@ -105,8 +118,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -117,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index 38d31dd5..560a0bda 100644 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -50,6 +52,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba default: } + s.requests++ resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { @@ -70,8 +73,12 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba src := string(body) for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } match := reNext.FindStringSubmatch(src) @@ -93,8 +100,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -106,5 +117,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/submd/submd.go b/pkg/subscraping/sources/submd/submd.go index 35291938..3d4de804 100644 --- a/pkg/subscraping/sources/submd/submd.go +++ b/pkg/subscraping/sources/submd/submd.go @@ -15,12 +15,14 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -28,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := s.fetch(ctx, domain, session) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -41,8 +44,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for sc.Scan() { if line := sc.Text(); line != "" { for _, sub := range session.Extractor.Extract(line) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: sub}: + s.results++ + } } } } @@ -70,13 +77,16 @@ func (s *Source) fetch(ctx context.Context, domain string, session *subscraping. func (s *Source) Name() string { return "submd" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } -func (s *Source) NeedsKey() bool { return false } -func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.OptionalKey } +func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } +func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, } } diff --git a/pkg/subscraping/sources/thc/thc.go b/pkg/subscraping/sources/thc/thc.go new file mode 100644 index 00000000..6edd2983 --- /dev/null +++ b/pkg/subscraping/sources/thc/thc.go @@ -0,0 +1,135 @@ +// Package thc logic +package thc + +import ( + "bytes" + "context" + "encoding/json" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Domains []struct { + Domain string `json:"domain"` + } `json:"domains"` + NextPageState string `json:"next_page_state"` +} + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +type requestBody struct { + Domain string `json:"domain"` + PageState string `json:"page_state"` + Limit int `json:"limit"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + var pageState string + headers := map[string]string{"Content-Type": "application/json"} + apiURL := "https://ip.thc.org/api/v1/lookup/subdomains" + + for { + reqBody := requestBody{ + Domain: domain, + PageState: pageState, + Limit: 1000, + } + + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + s.requests++ + resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var thcResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, domainRecord := range thcResponse.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain} + s.results++ + } + + pageState = thcResponse.NextPageState + + if pageState == "" { + break + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "thc" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(_ []string) { + // No API keys needed for THC +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go index 0033a4d3..3871920a 100644 --- a/pkg/subscraping/sources/threatbook/threatbook.go +++ b/pkg/subscraping/sources/threatbook/threatbook.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -51,6 +53,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -87,8 +90,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if total > 0 { for _, subdomain := range response.Data.SubDomains.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -109,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -123,5 +134,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index e09ab91e..897c7334 100644 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -23,6 +23,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run queries the ThreatCrowd API for the given domain and returns found subdomains. @@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func(startTime time.Time) { defer func() { @@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.Client.Do(req) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -80,8 +83,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for _, subdomain := range tcResponse.Subdomains { if subdomain != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }(time.Now()) @@ -104,9 +111,14 @@ func (s *Source) HasRecursiveSupport() bool { return false } +// KeyRequirement returns the API key requirement level for this source. +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + // NeedsKey indicates if the source requires an API key. func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys is a no-op since ThreatCrowd does not require an API key. @@ -118,5 +130,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 6776f9ef..d7224c8e 100644 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -22,6 +22,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -29,6 +30,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -36,6 +38,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -55,8 +58,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Results { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } }() @@ -76,8 +83,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -89,5 +100,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go new file mode 100644 index 00000000..1b0a9fcf --- /dev/null +++ b/pkg/subscraping/sources/urlscan/urlscan.go @@ -0,0 +1,215 @@ +// Package urlscan logic +package urlscan + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the URLScan API base URL + baseURL = "https://urlscan.io/api/v1/search/" + // maxPages is the maximum number of pages to fetch + maxPages = 5 + // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) + maxPerPage = 100 +) + +// response represents the URLScan API response structure +type response struct { + Results []struct { + Task struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"task"` + Page struct { + Domain string `json:"domain"` + URL string `json:"url"` + } `json:"page"` + Sort []interface{} `json:"sort"` + } `json:"results"` + HasMore bool `json:"has_more"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + s.skipped = false + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"api-key": randomApiKey} + + // Search with wildcard to get more subdomain results + s.enumerate(ctx, domain, headers, session, results) + }() + + return results +} + +// enumerate performs the actual enumeration with pagination +func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { + var searchAfter string + currentPage := 0 + + for { + select { + case <-ctx.Done(): + return + default: + } + + if currentPage >= maxPages { + break + } + + // Build search URL + searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) + if searchAfter != "" { + searchURL += "&search_after=" + url.QueryEscape(searchAfter) + } + + s.requests++ + resp, err := session.Get(ctx, searchURL, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + session.DiscardHTTPResponse(resp) + + // Process results - extract subdomains from multiple fields + for _, result := range data.Results { + candidates := []string{ + result.Task.Domain, + result.Page.Domain, + } + + // Also extract from URLs if present + if result.Task.URL != "" { + if u, err := url.Parse(result.Task.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + if result.Page.URL != "" { + if u, err := url.Parse(result.Page.URL); err == nil { + candidates = append(candidates, u.Hostname()) + } + } + + for _, candidate := range candidates { + if candidate == "" { + continue + } + for _, subdomain := range session.Extractor.Extract(candidate) { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } + } + } + } + + // Check pagination conditions + if !data.HasMore || len(data.Results) == 0 { + break + } + + // Get sort value for next page + lastResult := data.Results[len(data.Results)-1] + if len(lastResult.Sort) == 0 { + break + } + + // Build search_after parameter + sortValues := make([]string, len(lastResult.Sort)) + for i, v := range lastResult.Sort { + switch val := v.(type) { + case float64: + sortValues[i] = fmt.Sprintf("%.0f", val) + default: + sortValues[i] = fmt.Sprintf("%v", v) + } + } + searchAfter = strings.Join(sortValues, ",") + currentPage++ + } +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "urlscan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + Requests: s.requests, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 7ac4c6c5..812c7c9c 100644 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -30,6 +30,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -38,6 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -51,10 +53,16 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var cursor = "" for { + select { + case <-ctx.Done(): + return + default: + } var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } + s.requests++ resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -78,8 +86,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for _, subdomain := range data.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}: + s.results++ + } } cursor = data.Meta.Cursor if cursor == "" { @@ -104,8 +116,12 @@ func (s *Source) HasRecursiveSupport() bool { return true } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -116,6 +132,7 @@ func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, + Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index f3cba980..25687659 100644 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -17,6 +17,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int } // Run function returns all subdomains found with the service @@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -31,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -43,19 +46,27 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { + select { + case <-ctx.Done(): + return + default: + } line := scanner.Text() if line == "" { continue } line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { - // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: + s.results++ + } } } }() @@ -76,8 +87,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.NoKey +} + func (s *Source) NeedsKey() bool { - return false + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { @@ -89,5 +104,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index 18375e3f..284b920a 100644 --- a/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -33,6 +33,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -41,6 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -54,6 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } + s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -74,8 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } } }() @@ -95,8 +102,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -109,5 +120,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/sources/windvane/windvane.go b/pkg/subscraping/sources/windvane/windvane.go new file mode 100644 index 00000000..36a27dc6 --- /dev/null +++ b/pkg/subscraping/sources/windvane/windvane.go @@ -0,0 +1,162 @@ +// Package windvane logic +package windvane + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data responseData `json:"data"` +} + +type responseData struct { + List []domainEntry `json:"list"` + PageResponse pageInfo `json:"page_response"` +} + +type domainEntry struct { + Domain string `json:"domain"` +} + +type pageInfo struct { + Total string `json:"total"` + Count string `json:"count"` + TotalPage string `json:"total_page"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + requests int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + s.requests = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-Api-Key": randomApiKey} + + page := 1 + count := 1000 + for { + select { + case <-ctx.Done(): + return + default: + } + var resp *http.Response + var err error + + requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) + s.requests++ + resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", + "", headers, bytes.NewReader(requestBody)) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + defer session.DiscardHTTPResponse(resp) + + var windvaneResponse response + err = json.NewDecoder(resp.Body).Decode(&windvaneResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + for _, record := range windvaneResponse.Data.List { + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: + s.results++ + } + } + + pageInfo := windvaneResponse.Data.PageResponse + var totalRecords, recordsPerPage int + + if totalRecords, err = strconv.Atoi(pageInfo.Total); err != nil { + break + } + if recordsPerPage, err = strconv.Atoi(pageInfo.Count); err != nil { + break + } + + if (page-1)*recordsPerPage >= totalRecords { + break + } + + page++ + } + + }() + + return results +} + +func (s *Source) Name() string { + return "windvane" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + +func (s *Source) NeedsKey() bool { + return s.KeyRequirement() == subscraping.RequiredKey +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + Requests: s.requests, + } +} diff --git a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go index 62c2d1f9..61aa7197 100644 --- a/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go +++ b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -27,6 +27,7 @@ type Source struct { timeTaken time.Duration errors int results int + requests int skipped bool } @@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results := make(chan subscraping.Result) s.errors = 0 s.results = 0 + s.requests = 0 go func() { defer func(startTime time.Time) { @@ -63,7 +65,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { + select { + case <-ctx.Done(): + return + default: + } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) + s.requests++ resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { @@ -87,8 +95,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se _ = resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} - s.results++ + select { + case <-ctx.Done(): + return + case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}: + s.results++ + } } } }() @@ -109,8 +121,12 @@ func (s *Source) HasRecursiveSupport() bool { return false } +func (s *Source) KeyRequirement() subscraping.KeyRequirement { + return subscraping.RequiredKey +} + func (s *Source) NeedsKey() bool { - return true + return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { @@ -123,5 +139,6 @@ func (s *Source) Statistics() subscraping.Statistics { Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, + Requests: s.requests, } } diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 454ad776..38118717 100644 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -16,7 +16,8 @@ const ( ) type CustomRateLimit struct { - Custom mapsutil.SyncLockMap[string, uint] + Custom mapsutil.SyncLockMap[string, uint] + CustomDuration mapsutil.SyncLockMap[string, time.Duration] } // BasicAuth request's Authorization header @@ -28,11 +29,21 @@ type BasicAuth struct { // Statistics contains statistics about the scraping process type Statistics struct { TimeTaken time.Duration + Requests int Errors int Results int Skipped bool } +// KeyRequirement represents the API key requirement level for a source +type KeyRequirement int + +const ( + NoKey KeyRequirement = iota + OptionalKey + RequiredKey +) + // Source is an interface inherited by each passive source type Source interface { // Run takes a domain as argument and a session object @@ -52,7 +63,11 @@ type Source interface { // not just root domains. HasRecursiveSupport() bool - // NeedsKey returns true if the source requires an API key + // KeyRequirement returns the API key requirement level for this source + KeyRequirement() KeyRequirement + + // NeedsKey returns true if the source requires an API key. + // Deprecated: Use KeyRequirement() instead for more granular control. NeedsKey() bool AddApiKeys([]string) @@ -75,6 +90,8 @@ type Session struct { Client *http.Client // Rate limit instance MultiRateLimiter *ratelimit.MultiLimiter + // Timeout is the timeout in seconds for requests + Timeout int } // Result is a result structure returned by a source