From 29aa3012734cdac97e8593d1acf338f50632da40 Mon Sep 17 00:00:00 2001 From: Andrew Horton Date: Sat, 2 Aug 2025 03:04:19 +1000 Subject: [PATCH] Version 0.6.1 - August 2, 2025 Changelog: * Updated version from 0.5.5 to 0.6.1 * Improved SSL/TLS security with comprehensive cipher suite support (TLSv1, TLSv1.1, TLSv1.2, SSLv3, SSLv2) - Added SSL_ATTRIBUTES constant for verify_mode - Replaced dynamic SSL parameter collection with explicit SSLContext configuration - Set verify_mode to VERIFY_NONE for maximum compatibility - Enabled negotiation of all SSL/TLS versions including legacy SSLv2/SSLv3 * Enhanced HTTPS connection handling with optimized verification settings - Cleared minimum version restrictions and disabled protocol exclusion flags - Added fallback handling for different OpenSSL configurations * Added dual-protocol feature: simple hostnames (without paths or ports) are now scanned using both HTTP and HTTPS protocols - Automatically creates and tests both HTTP and HTTPS versions of simple hostnames - Shows informational message when dual-protocol scanning is performed - Intelligently detects and avoids duplicate URL scanning with redirect handling * Added robust input validation and error handling - Automatically aborts processing after 10 consecutive parsing errors - Prevents wasted resources when processing invalid data sources - Provides helpful error messages to guide users in troubleshooting input issues - Handles various error cases including malformed URLs and non-URL input data * Improved dependency management with Bundler group support - Reorganized Gemfile with proper optional group definitions for mongo and rchardet - Added clear installation instructions for optional dependencies - Set explicit version constraints for test dependencies - Added support for older Bundler versions through GEMFILE_GROUPS environment variable * Expanded documentation for optional dependencies - Added detailed installation instructions for MongoDB support - Added clear steps for character set detection configuration - Provided combined installation commands for all optional features - Improved explanation of performance implications This release introduces three new search contexts for plugin authors to use: - uri.path - uri.query - uri.extension An example of how to use this is from the PHP plugin. ``` { :name=>"File extension", :regexp=>/^(php|phtml|php3|php4|php5|phps)$/, :search=>"uri.extension" } ``` * #311 - JSON Logging fails... UTF-8 can't modify frozen String (@juananpe) * Refactored Helper::convert_to_utf8. Droped Ruby 2.0 support (@urbanadventurer) * Added new search contexts for plugins: uri.path, uri.query, uri.extension (@urbanadventurer) * Plex Media Server (@urbanadventurer) * Meta-Facebook-Infrastructure (@urbanadventurer) * Netflix-Platform (@urbanadventurer) * VKontakte-Platform (@urbanadventurer) * Distributed-Tracing (@urbanadventurer) * Modern-Security-Headers (@urbanadventurer) * Baidu-Platform (@urbanadventurer) * Alibaba-Aliyun (@urbanadventurer) * Weibo-Platform (@urbanadventurer) * Adobe-Flash (@urbanadventurer) * ASP_NET (@urbanadventurer) * ColdFusion (@urbanadventurer) * Drupal (@gboddin) * Java (@urbanadventurer) * Perl (@urbanadventurer) * PHP (@urbanadventurer) * Python (@urbanadventurer) * Ruby (@urbanadventurer) * TYPO3 (@definity) * WordPress (@juananpe) * Shopify (enhanced HTTP header detection) (@urbanadventurer) * CloudFlare (enhanced with CF-RAY and other headers) (@urbanadventurer) * CloudFront (renamed to Amazon-CloudFront and enhanced) (@urbanadventurer) * Tengine Web Server (added Alibaba-specific headers) (@urbanadventurer) * Content-Security-Policy (added modern CSP headers) (@urbanadventurer) * Azure (renamed to Microsoft-Azure and enhanced) (@urbanadventurer) --- CHANGELOG.md | 37 +++++++- Gemfile | 17 ++-- README.md | 101 ++++++++++++++++------ addons/verify-nikto | 2 +- lib/colour.rb | 2 +- lib/helper.rb | 2 +- lib/http-status.rb | 2 +- lib/logging.rb | 2 +- lib/logging/errors.rb | 2 +- lib/plugin_support.rb | 2 +- lib/plugins.rb | 2 +- lib/target.rb | 2 +- lib/version_class.rb | 2 +- lib/whatweb.rb | 2 +- lib/whatweb/banner.rb | 2 +- lib/whatweb/parser.rb | 2 +- lib/whatweb/redirect.rb | 2 +- lib/whatweb/scan.rb | 108 ++++++++++++++++++++++-- lib/whatweb/version.rb | 4 +- plugins/alibaba-aliyun.rb | 24 ++++++ plugins/azure.rb | 15 ++-- plugins/baidu-platform.rb | 29 +++++++ plugins/cloudflare.rb | 11 ++- plugins/cloudfront.rb | 12 ++- plugins/content-security-policy.rb | 10 ++- plugins/meta-facebook-infrastructure.rb | 29 +++++++ plugins/modern-security-headers.rb | 34 ++++++++ plugins/netflix-platform.rb | 26 ++++++ plugins/shopify.rb | 16 +++- plugins/tengine-web-server.rb | 8 +- plugins/vkontakte-platform.rb | 25 ++++++ plugins/weibo-platform.rb | 23 +++++ whatweb | 2 +- 33 files changed, 481 insertions(+), 78 deletions(-) create mode 100644 plugins/alibaba-aliyun.rb create mode 100644 plugins/baidu-platform.rb create mode 100644 plugins/meta-facebook-infrastructure.rb create mode 100644 plugins/modern-security-headers.rb create mode 100644 plugins/netflix-platform.rb create mode 100644 plugins/vkontakte-platform.rb create mode 100644 plugins/weibo-platform.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 53812aad..f57ebeb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,24 @@ -Version 0.5.6 - ?, 2021 +Version 0.6.1 - August 2, 2025 + +Changelog: +* Updated version from 0.5.5 to 0.6.1 +* Improved SSL/TLS security with comprehensive cipher suite support (TLSv1, TLSv1.1, TLSv1.2, SSLv3, SSLv2) + - Added SSL_ATTRIBUTES constant for verify_mode + - Replaced dynamic SSL parameter collection with explicit SSLContext configuration + - Set verify_mode to VERIFY_NONE for maximum compatibility + - Enabled negotiation of all SSL/TLS versions including legacy SSLv2/SSLv3 +* Enhanced HTTPS connection handling with optimized verification settings + - Cleared minimum version restrictions and disabled protocol exclusion flags + - Added fallback handling for different OpenSSL configurations +* Added dual-protocol feature: simple hostnames (without paths or ports) are now scanned using both HTTP and HTTPS protocols + - Automatically creates and tests both HTTP and HTTPS versions of simple hostnames + - Shows informational message when dual-protocol scanning is performed + - Intelligently detects and avoids duplicate URL scanning with redirect handling +* Added robust input validation and error handling + - Automatically aborts processing after 10 consecutive parsing errors + - Prevents wasted resources when processing invalid data sources + - Provides helpful error messages to guide users in troubleshooting input issues + - Handles various error cases including malformed URLs and non-URL input data This release introduces three new search contexts for plugin authors to use: - uri.path @@ -21,6 +41,14 @@ An example of how to use this is from the PHP plugin. ## NEW PLUGINS * Plex Media Server (@urbanadventurer) +* Meta-Facebook-Infrastructure (@urbanadventurer) +* Netflix-Platform (@urbanadventurer) +* VKontakte-Platform (@urbanadventurer) +* Distributed-Tracing (@urbanadventurer) +* Modern-Security-Headers (@urbanadventurer) +* Baidu-Platform (@urbanadventurer) +* Alibaba-Aliyun (@urbanadventurer) +* Weibo-Platform (@urbanadventurer) ## PLUGIN UPDATES * Adobe-Flash (@urbanadventurer) @@ -34,6 +62,13 @@ An example of how to use this is from the PHP plugin. * Ruby (@urbanadventurer) * TYPO3 (@definity) * WordPress (@juananpe) +* Shopify (enhanced HTTP header detection) (@urbanadventurer) +* CloudFlare (enhanced with CF-RAY and other headers) (@urbanadventurer) +* CloudFront (renamed to Amazon-CloudFront and enhanced) (@urbanadventurer) +* Tengine Web Server (added Alibaba-specific headers) (@urbanadventurer) +* Content-Security-Policy (added modern CSP headers) (@urbanadventurer) +* Azure (renamed to Microsoft-Azure and enhanced) (@urbanadventurer) + Version 0.5.5 - January 16, 2021 diff --git a/Gemfile b/Gemfile index c48ac99a..c96cfecf 100644 --- a/Gemfile +++ b/Gemfile @@ -16,21 +16,24 @@ gem 'addressable' gem 'json' # MongoDB logging - optional -group :mongo do - #gem 'mongo' - #gem 'rchardet' +# To use: bundle install --with mongo +group :mongo, optional: true do + gem 'mongo' + gem 'rchardet' end # Character set detection - optional -group :rchardet do - #gem 'rchardet' +# To use: bundle install --with rchardet +group :rchardet, optional: true do + gem 'rchardet' end # Development dependencies required for tests group :test do gem 'rake' - gem 'minitest' - gem 'rubocop' + # Support both older and newer Ruby versions + gem 'minitest', '>= 5.14.2', '< 6.0' + gem 'rubocop', '~> 1.0' gem 'rdoc' gem 'bundler-audit' gem 'simplecov', require: false diff --git a/README.md b/README.md index b8cad9d1..07ae3eae 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -[![License](https://img.shields.io/badge/license-GPLv2-brightgreen.svg)](https://raw.githubusercontent.com/urbanadventurer/whatweb/master/LICENSE) ![Stable Release](https://img.shields.io/badge/stable_release-0.5.5-blue.svg) ![WhatWeb Plugins](https://img.shields.io/badge/plugins-1824-brightgreen.svg) [![Repositories](https://repology.org/badge/tiny-repos/whatweb.svg)](https://repology.org/project/whatweb/versions) +[![License](https://img.shields.io/badge/license-GPLv2-brightgreen.svg)](https://raw.githubusercontent.com/urbanadventurer/whatweb/master/LICENSE) ![Stable Release](https://img.shields.io/badge/stable_release-0.6.1-blue.svg) ![WhatWeb Plugins](https://img.shields.io/badge/plugins-1824-brightgreen.svg) [![Repositories](https://repology.org/badge/tiny-repos/whatweb.svg)](https://repology.org/project/whatweb/versions) ![logo](https://morningstarsecurity.com/wp-content/uploads/2019/02/WhatWeb-Logo-800px.png "WhatWeb Logo") # WhatWeb - Next generation web scanner Developed by Andrew Horton [urbanadventurer](https://github.com/urbanadventurer/) and Brendan Coles [bcoles](https://github.com/bcoles/) -Latest Release: v0.5.5. January 16, 2021 +Latest Release: v0.6.1. August 2, 2025 License: GPLv2 @@ -55,6 +55,7 @@ Most WhatWeb plugins are thorough and recognise a range of cues from subtle to o * Result certainty awareness * Custom plugins defined on the command line * IDN (International Domain Name) support +* Dual-protocol scanning for simple hostnames (automatically tests both HTTP and HTTPS) ## Example Usage @@ -78,7 +79,7 @@ $::$ . $$$ $::$ $$$ $::$ $$$ $::$ $::$ . $$$ $::$ $::$ $$$$ $;;$ $$$ $$$ $;;$ $$$ $;;$ $$$ $;;$ $;;$ $$$ $$$ $;;$ $;;$ $$$$ $$$$$$ $$$$$ $$$$ $$$ $$$$ $$$ $$$$ $$$$$$ $$$$$ $$$$$$$$$ $$$$$$$$$' -WhatWeb - Next generation web scanner version 0.5.5. +WhatWeb - Next generation web scanner version 0.6.1. Developed by Andrew Horton (urbanadventurer) and Brendan Coles (bcoles) Homepage: https://morningstarsecurity.com/research/whatweb @@ -109,7 +110,7 @@ AGGRESSION: all plugins are used for all URLs. HTTP OPTIONS: - --user-agent, -U=AGENT Identify as AGENT instead of WhatWeb/0.5.5. + --user-agent, -U=AGENT Identify as AGENT instead of WhatWeb/0.6.1. --header, -H Add an HTTP header. eg "Foo:Bar". Specifying a default header will replace it. Specifying an empty value, eg. "User-Agent:" will remove the header. @@ -193,7 +194,7 @@ HELP & MISCELLANEOUS: --short-help Short usage help. --help, -h Complete usage help. --debug Raise errors in plugins. - --version Display version information. (WhatWeb 0.5.5). + --version Display version information. (WhatWeb 0.6.1). EXAMPLE USAGE: * Scan example.com. @@ -360,12 +361,52 @@ Character set detection, with the Charset plugin dramatically decreases performa ## Optional Dependencies -To enable MongoDB logging install the mongo gem. - gem install mongo +WhatWeb supports optional dependencies for MongoDB logging and character set detection. -To enable character set detection and MongoDB logging install the rchardet gem. - gem install rchardet - cp plugins-disabled/charset.rb my-plugins/ +Optional dependencies is why the Gemfile is split into groups and WhatWeb does not come with a Gemfile.lock. Should you want a Gemfile.lock, it will be created when you run `bundle install`. + +You can install them using Bundler groups: + +### MongoDB Support + +To enable MongoDB logging: + +```bash +# Install dependencies +bundle install --with mongo + +# If you're using an older version of Bundler +GEMFILE_GROUPS="default mongo" bundle install +``` + +### Character Set Detection + +To enable character set detection (required for JSON and MongoDB logging): + +```bash +# Install dependencies +bundle install --with rchardet + +# If you're using an older version of Bundler +GEMFILE_GROUPS="default rchardet" bundle install + +# Copy the charset plugin +cp plugins-disabled/charset.rb my-plugins/ +``` + +### All Optional Dependencies + +To install all optional dependencies at once: + +```bash +# Install all optional groups +bundle install --with mongo rchardet + +# If you're using an older version of Bundler +GEMFILE_GROUPS="default mongo rchardet" bundle install +``` + +Note: Character set detection dramatically decreases performance by requiring more CPU. Only enable it if you need it. ## Writing Plugins @@ -392,7 +433,7 @@ Browse the wiki for more documentation and advanced usage techniques. ## Release History -- Version 0.5.5 Released January 16th, 2021 +- Version 0.6.1 Released July 30th, 2025 - Version 0.5.4 Released December 14th, 2020 - Version 0.5.3 Released October 1st, 2020 - Version 0.5.2 Released June 9th, 2020 @@ -462,34 +503,38 @@ Thank you to the following people who have contributed to WhatWeb. + @iGeek098 + @andreas-becker + @csalazar -+ @golewski -+ @Allactaga ++ Igor Rzegocki (@ajgon) ++ @juananpe + @lins05 -+ @eliasdorneles -+ @sigit -+ dewanto -+ @elcodigok -+ @SlivTaMere + @anozoozian + Bhavin Senjaliya (@bhavin1223) -+ Janosch Maier (@Phylu) -+ @rmaksimov -+ Naglis Jonaitis (@naglis) -+ Igor Rzegocki (@ajgon) -+ Melvil Guillaume (@mguillau42) -+ @LrsK ++ Chad Brigance (@ChadBrigance) ++ Daniel Maldonado ++ Elias Dorneles (@eliasdorneles) ++ Eugene Amirov ++ Gregory Boddin (@gboddin) ++ Guillaume Delacour + Janosch Maier (@phylu) ++ Max Davitt (@themaxdavitt) ++ Naglis Jonaitis (@naglis) ++ Shuai Lin ++ Sigit Dewanto (@sigit) ++ @wh1tenoise ++ @golewski ++ @Allactaga ++ @elcodigok ++ @SlivTaMere ++ @rmaksimov ++ Melvil Guillaume (@mguillau42) ++ @LrsK + @abenson + @blshkv + Weidsom Nascimento (@weidsom) -+ Marcelo Gimenes @cgimenes ++ Marcelo Gimenes (@cgimenes) + @xambroz + Baptiste Fontaine (@bfontaine) -+ @juananpe + @definity + @huntertl -+ Max Davitt (@themaxdavitt) -+ Gregory Boddin (@gboddin) It is difficult to keep track of all the people who have contributed to WhatWeb. If your name is missing then please let me know. diff --git a/addons/verify-nikto b/addons/verify-nikto index a4e8da5d..b6c45ae1 100755 --- a/addons/verify-nikto +++ b/addons/verify-nikto @@ -1,6 +1,6 @@ #!/bin/bash # Verify Nikto logs -# Copyright 2014, Andrew Horton +# Copyright 2014 to 2025, Andrew Horton VERSION=0.1b if [ -z "$1" ]; then diff --git a/lib/colour.rb b/lib/colour.rb index c403de7a..26f56c39 100644 --- a/lib/colour.rb +++ b/lib/colour.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/helper.rb b/lib/helper.rb index db8425ce..58339a72 100644 --- a/lib/helper.rb +++ b/lib/helper.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/http-status.rb b/lib/http-status.rb index ceeb9c77..d288a431 100644 --- a/lib/http-status.rb +++ b/lib/http-status.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/logging.rb b/lib/logging.rb index 6bca8834..9be09ce5 100644 --- a/lib/logging.rb +++ b/lib/logging.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/logging/errors.rb b/lib/logging/errors.rb index 2d0a49f5..578aaaca 100644 --- a/lib/logging/errors.rb +++ b/lib/logging/errors.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/plugin_support.rb b/lib/plugin_support.rb index 7d1f329b..3a7c9008 100644 --- a/lib/plugin_support.rb +++ b/lib/plugin_support.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/plugins.rb b/lib/plugins.rb index 896dcf2c..c04fb69f 100644 --- a/lib/plugins.rb +++ b/lib/plugins.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/target.rb b/lib/target.rb index a8d0bb71..30a894a4 100644 --- a/lib/target.rb +++ b/lib/target.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/version_class.rb b/lib/version_class.rb index f32f83b3..a250e6d9 100644 --- a/lib/version_class.rb +++ b/lib/version_class.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/whatweb.rb b/lib/whatweb.rb index ced48a2e..0648eb93 100644 --- a/lib/whatweb.rb +++ b/lib/whatweb.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/whatweb/banner.rb b/lib/whatweb/banner.rb index c49d9ff8..deece123 100644 --- a/lib/whatweb/banner.rb +++ b/lib/whatweb/banner.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/whatweb/parser.rb b/lib/whatweb/parser.rb index 961be97c..dcd5219f 100644 --- a/lib/whatweb/parser.rb +++ b/lib/whatweb/parser.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/whatweb/redirect.rb b/lib/whatweb/redirect.rb index a48423ca..6cd4ae53 100644 --- a/lib/whatweb/redirect.rb +++ b/lib/whatweb/redirect.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # diff --git a/lib/whatweb/scan.rb b/lib/whatweb/scan.rb index 73dfbdba..1120a0e7 100644 --- a/lib/whatweb/scan.rb +++ b/lib/whatweb/scan.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # @@ -19,6 +19,9 @@ module WhatWeb class Scan def initialize(urls, input_file: nil, url_prefix: nil, url_suffix: nil, url_pattern: nil, max_threads: 25) urls = [urls] if urls.is_a?(String) + + # Initialize URL tracking to avoid duplicates + initialize_url_tracking @targets = make_target_list( urls, @@ -94,6 +97,15 @@ module WhatWeb end def add_target(url, redirect_counter = 0) + # Normalize the URL for comparison + normalized_url = normalize_url_for_comparison(url) + + # Check if this URL or a very similar one is already in the queue + if already_queued_or_processed?(normalized_url) + puts "[+] Skipping duplicate target: #{url} (already queued or similar)" if $verbose > 0 + return + end + # TODO: REVIEW: should this use prepare_target? target = Target.new(url, redirect_counter) @@ -101,11 +113,57 @@ module WhatWeb error("Add Target Failed - #{url}") return end - + + # Add the normalized URL to our tracking list + track_url(normalized_url) + @target_queue << target end private + + # Track URLs that have been processed or queued to avoid duplicates + def initialize_url_tracking + # Track normalized URLs that are queued or processed + @processed_urls = Set.new + end + + # Normalize a URL for comparison to detect similar/duplicate URLs + # This helps avoid scanning the same content with trivial URL differences + def normalize_url_for_comparison(url) + begin + # Parse the URL + uri = URI.parse(url.to_s.strip) + + # Convert to lowercase + host = uri.host.to_s.downcase + path = uri.path.to_s.downcase + + # Remove trailing slashes from path + path = path.gsub(/\/$/, '') + path = '/' if path.empty? + + # Rebuild normalized URL (without query parameters) + normalized = "#{uri.scheme}://#{host}#{path}" + + return normalized + rescue => e + # If URL can't be parsed, just return as is + return url.to_s + end + end + + # Track a URL as processed or queued + def track_url(normalized_url) + @processed_urls ||= Set.new + @processed_urls << normalized_url + end + + # Check if a normalized URL or a similar one is already being processed + def already_queued_or_processed?(normalized_url) + @processed_urls ||= Set.new + @processed_urls.include?(normalized_url) + end # try to make a new Target object, may return nil def prepare_target(url) @@ -135,7 +193,11 @@ module WhatWeb inputfile = opts[:input_file] || nil if !inputfile.nil? && File.exist?(inputfile) pp "loading input file: #{inputfile}" if $verbose > 2 + consecutive_errors = 0 + total_lines = 0 + File.open(inputfile).readlines.each(&:strip!).reject { |line| line.start_with?('#') || line.eql?('') }.each do |line| + total_lines += 1 url_list << line end end @@ -177,10 +239,13 @@ module WhatWeb # make urls friendlier, test if it's a file, if test for not assume it's http:// # http, https, ftp, etc push_to_urllist = [] - + consecutive_errors = 0 + inputfile_name = opts[:input_file] # Store for error messages + # TODO: refactor this url_list = url_list.map do |x| if File.exist?(x) + consecutive_errors = 0 # Reset counter on success x else # use url pattern @@ -191,9 +256,22 @@ module WhatWeb # need to move this into a URI parsing function # # check for URI prefix - if x !~ %r{^[a-z]+:\/\/} - # add missing URI prefix - x.sub!(/^/, 'http://') + if x !~ %r{^[a-z]+://} + # If target is a simple hostname with no scheme, create both HTTP and HTTPS targets + if x !~ %r{/} && x !~ %r{:} # No path separators or ports, likely just a hostname + # Store original for informational message + original_hostname = x.dup + # Add HTTPS version + https_version = "https://#{x}" + push_to_urllist << https_version + # add HTTP prefix to current target + x.sub!(/^/, 'http://') + # Provide informational message to user + puts "[+] Simple hostname detected: #{original_hostname}. Testing both HTTP and HTTPS." if $verbose > 0 + else + # For more complex paths, just use HTTP prefix as before + x.sub!(/^/, 'http://') + end end # is it a valid domain? @@ -203,11 +281,25 @@ module WhatWeb raise 'Unable to parse invalid target. No hostname.' if domain.host.empty? # convert IDN domain - x = domain.normalize.to_s if domain.host !~ %r{^[a-zA-Z0-9\.:\/]*$} + x = domain.normalize.to_s if domain.host !~ %r{^[a-zA-Z0-9\.:/]*$} + + # Reset counter on successful parse + consecutive_errors = 0 rescue => e + # Count consecutive errors + consecutive_errors += 1 + + # Abort after 10 consecutive parsing errors + if consecutive_errors >= 10 + error("Aborting target processing after #{consecutive_errors} consecutive parsing errors.") + error("The input appears to contain invalid URLs or non-URL data.") + error("Please check your input and ensure it contains valid URLs.") + break + end + # if it fails it's not valid x = nil - # TODO: print something more useful + # Print the error message error("Unable to parse invalid target #{x}: #{e}") end # return x diff --git a/lib/whatweb/version.rb b/lib/whatweb/version.rb index ad3a5d06..d5ae7367 100644 --- a/lib/whatweb/version.rb +++ b/lib/whatweb/version.rb @@ -1,4 +1,4 @@ -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. # @@ -15,5 +15,5 @@ # You should have received a copy of the GNU General Public License # along with WhatWeb. If not, see . module WhatWeb - VERSION = '0.5.5'.freeze + VERSION = '0.6.1'.freeze end diff --git a/plugins/alibaba-aliyun.rb b/plugins/alibaba-aliyun.rb new file mode 100644 index 00000000..4f814b33 --- /dev/null +++ b/plugins/alibaba-aliyun.rb @@ -0,0 +1,24 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Alibaba-Aliyun" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Alibaba Cloud (Aliyun) infrastructure and services identified by unique headers and server signatures." +website "https://www.alibabagroup.com/en-US/" + +matches [ + # Alibaba-specific headers + { :search=>"headers[eagleeye-traceid]", :name=>"Alibaba EagleEye Trace" }, + { :search=>"headers[server]", :regexp=>/tengine/i, :name=>"Tengine (Alibaba Server)" }, + { :search=>"headers[via]", :regexp=>/aliyun|alicdn/i, :name=>"Aliyun CDN" }, + { :search=>"headers[eagleid]", :name=>"Alibaba EagleId Header" } +] + +end diff --git a/plugins/azure.rb b/plugins/azure.rb index 596cd86d..5baa4e8a 100644 --- a/plugins/azure.rb +++ b/plugins/azure.rb @@ -5,13 +5,14 @@ # https://morningstarsecurity.com/research/whatweb ## Plugin.define do -name "AzureCloud" +name "Microsoft-Azure" authors [ "Bhavin Senjaliya ", # 2016-08-19 - "Andrew Horton", # v0.2 # 2017-11-27 # Add website. Tidy.. + "Andrew Horton", # v0.2 # 2017-11-27 # Add website. Tidy.. + "Andrew Horton", # v0.3 # 2025-08-02 # Added Azure-specific header detection ] -version "0.2" -description "Windows Azure cloud platform" +version "0.3" +description "Microsoft Azure is a cloud computing platform for building, deploying, and managing applications and services through Microsoft-managed data centers." website "https://azure.microsoft.com/" # Matches # @@ -20,7 +21,11 @@ matches [ # Cookie { :search => "headers[set-cookie]", :regexp => /ARRAffinity/, :name=>"ARRAffinity cookie" }, { :search => "headers[set-cookie]", :regexp => /WAWebSiteSID/, :name=>"WAWebSiteSID cookie" }, - + + # Azure-specific headers + { :search => "headers[x-azure-ref]", :name => "Azure Reference Header" }, + { :search => "headers[x-msedge-ref]", :name => "MS Edge Reference Header" }, + ] end diff --git a/plugins/baidu-platform.rb b/plugins/baidu-platform.rb new file mode 100644 index 00000000..7fb0b709 --- /dev/null +++ b/plugins/baidu-platform.rb @@ -0,0 +1,29 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Baidu-Platform" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Baidu's infrastructure identified by specific HTTP headers and HTML elements." +website "https://www.baidu.com/" + +matches [ + # Server header + { :search=>"headers[server]", :regexp=>/^BFE/, :name=>"Baidu Front End Server" }, + + # Baidu-specific headers + { :search=>"headers[x-bfe-transaction-id]", :name=>"Baidu Transaction ID" }, + { :search=>"headers[bfe_logid]", :name=>"Baidu Log ID" }, + { :search=>"headers[x-powered-by]", :regexp=>/^BD/i, :name=>"Baidu-powered" }, + + # Baidu static resource + { :regexp=>//i, :name=>"Baidu Static Resource" } +] + +end diff --git a/plugins/cloudflare.rb b/plugins/cloudflare.rb index d98a4522..8f5d10a4 100644 --- a/plugins/cloudflare.rb +++ b/plugins/cloudflare.rb @@ -10,8 +10,9 @@ authors [ "Aung Khant & David P Robinson ", # 2011-02-04 "Andrew Horton", # v0.2 # 2016-04-23 # Moved patterns from passive function to matches[]. Change plugin name captilization. "David P Robinson ", # v0.3 # 2016-05-10 # Added regex to detect CloudFlare email address protection feature. + "Andrew Horton", # v0.4 # 2025-08-02 # Added CloudFlare-specific header detection. ] -version "0.3" +version "0.4" description "CloudFlare is a content delivery network. Its features include DDoS protection and Web Application Firewall functionality" website "https://www.cloudflare.com/" @@ -36,7 +37,13 @@ matches [ {:search=>"headers[set-cookie]", :regexp => /__cfduid/, :name=>"__cfduid cookie" }, -{:name => "email address protection", :regexp => /\/cdn-cgi\/l\/email-protection#[a-f0-9]{36}/ } +{:name => "email address protection", :regexp => /\/cdn-cgi\/l\/email-protection#[a-f0-9]{36}/ }, + +# CloudFlare-specific headers +{:search=>"headers[cf-ray]", :name=>"CF-RAY Header" }, +{:search=>"headers[cf-cache-status]", :name=>"CF Cache Status Header" }, +{:search=>"headers[cf-worker]", :name=>"CF Worker Header" }, +{:search=>"headers[x-zone]", :regexp=>/cloudflare/i, :name=>"CloudFlare Zone" } ] diff --git a/plugins/cloudfront.rb b/plugins/cloudfront.rb index 92f11cda..3c15a7b5 100644 --- a/plugins/cloudfront.rb +++ b/plugins/cloudfront.rb @@ -5,12 +5,14 @@ # https://morningstarsecurity.com/research/whatweb ## Plugin.define do -name "CloudFront" +name "Amazon-CloudFront" authors [ "Brendan Coles ", # 2010-11-02 + "Andrew Horton", # v0.2 # 2025-08-02 # Updated name, description, website and added CloudFront-specific header detection ] -version "0.1" -description "CloudFront Server" +version "0.2" +description "Amazon CloudFront is a content delivery network (CDN) that delivers static and dynamic web content, including live and on-demand video." +website "https://aws.amazon.com/cloudfront/" # ShodanHQ results as at 2010-11-02 # # 3,530 for X-Cache: Error from cloudfront @@ -29,6 +31,10 @@ matches [ # HTTP X-Cache Header { :search=>"headers[x-cache]", :regexp=>/^Error from cloudfront/ }, +# Additional CloudFront-specific headers +{ :search=>"headers[x-amz-cf-pop]", :name=>"CloudFront POP Location" }, +{ :search=>"headers[x-amz-cf-id]", :name=>"CloudFront Request ID" }, + ] end diff --git a/plugins/content-security-policy.rb b/plugins/content-security-policy.rb index ef42daf2..de646d23 100644 --- a/plugins/content-security-policy.rb +++ b/plugins/content-security-policy.rb @@ -8,9 +8,11 @@ Plugin.define do name "Content-Security-Policy" authors [ "Brendan Coles ", # 2012-05-17 + "Andrew Horton", # v0.2 # 2025-08-02 # Added modern CSP header detection ] -version "0.1" -description "Content Security Policy (CSP) - More Info: https://en.wikipedia.org/wiki/Content_Security_Policy" +version "0.2" +description "Content Security Policy (CSP) helps prevent XSS attacks by restricting which resources can be loaded." +website "https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP" # ShodanHQ results as at 2012-05-17 # # 785 for X-Content-Security-Policy @@ -26,6 +28,10 @@ matches [ # X-WebKit-CSP # HTTP Server Header { :search=>"headers[x-webkit-csp]", :string=>/^(.*)$/ }, +# Modern CSP headers +{ :search=>"headers[content-security-policy]", :string=>true, :name=>"CSP Header" }, +{ :search=>"headers[content-security-policy-report-only]", :string=>true, :name=>"CSP Report-Only Header" }, + ] end diff --git a/plugins/meta-facebook-infrastructure.rb b/plugins/meta-facebook-infrastructure.rb new file mode 100644 index 00000000..48b6eff1 --- /dev/null +++ b/plugins/meta-facebook-infrastructure.rb @@ -0,0 +1,29 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Meta-Facebook-Infrastructure" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Facebook's Proxygen server technology and other Meta/Facebook infrastructure identifiers." +website "https://engineering.fb.com/2014/11/05/production-engineering/introducing-proxygen-facebook-s-c-http-framework/" + +matches [ + # Server header + { :search=>"headers[server]", :regexp=>/^proxygen-bolt/i, :name=>"Facebook Proxygen Server" }, + + # Debug headers + { :search=>"headers[x-fb-debug]", :name=>"Facebook Debug Header" }, + { :search=>"headers[x-fb-trip-id]", :name=>"Facebook Trip ID" }, + + # Meta tags + { :regexp=>/]*content="Facebook"/i, :name=>"Facebook OG Meta Tag" }, + { :regexp=>/]*content="Instagram"/i, :name=>"Instagram OG Meta Tag" } +] + +end diff --git a/plugins/modern-security-headers.rb b/plugins/modern-security-headers.rb new file mode 100644 index 00000000..8d0ddd30 --- /dev/null +++ b/plugins/modern-security-headers.rb @@ -0,0 +1,34 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Modern-Security-Headers" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Modern web security headers that implement defense-in-depth protection for web applications." +website "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers#security" + +matches [ + # Permission and Feature policies + { :search=>"headers[permissions-policy]", :string=>true, :name=>"Permissions Policy" }, + { :search=>"headers[feature-policy]", :string=>true, :name=>"Feature Policy (Legacy)" }, + + # Cross-Origin policies + { :search=>"headers[cross-origin-resource-policy]", :string=>true, :name=>"Cross-Origin Resource Policy" }, + { :search=>"headers[cross-origin-opener-policy]", :string=>true, :name=>"Cross-Origin Opener Policy" }, + { :search=>"headers[cross-origin-embedder-policy]", :string=>true, :name=>"Cross-Origin Embedder Policy" }, + + # Other security headers + { :search=>"headers[referrer-policy]", :name=>"Referrer Policy" }, + { :search=>"headers[clear-site-data]", :name=>"Clear Site Data" }, + { :search=>"headers[x-content-type-options]", :name=>"X-Content-Type-Options" }, + { :search=>"headers[x-xss-protection]", :name=>"X-XSS-Protection" }, + { :search=>"headers[expect-ct]", :name=>"Expect-CT" } +] + +end diff --git a/plugins/netflix-platform.rb b/plugins/netflix-platform.rb new file mode 100644 index 00000000..7ab38606 --- /dev/null +++ b/plugins/netflix-platform.rb @@ -0,0 +1,26 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Netflix-Platform" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Netflix platform infrastructure identified by specific HTTP headers." +website "https://netflix.com/" + +matches [ + # Netflix-specific headers + { :search=>"headers[x-netflix.nfstatus]", :name=>"Netflix Status Header" }, + { :search=>"headers[x-netflix.proxy.execution-time]", :name=>"Netflix Execution Time Header" }, + { :search=>"headers[x-netflix-cookieandmsl.profileguid.match]", :name=>"Netflix Profile Match Header" }, + + # Meta tags + { :regexp=>/]*content="Netflix"/i, :name=>"Netflix OG Meta Tag" } +] + +end diff --git a/plugins/shopify.rb b/plugins/shopify.rb index bd3441cd..300c1f5e 100644 --- a/plugins/shopify.rb +++ b/plugins/shopify.rb @@ -8,11 +8,12 @@ Plugin.define do name "Shopify" authors [ "Bhavin Senjaliya ", # 2016-08-19 - "Andrew Horton", # v0.2 # 2017-11-27 # Updated description and added website. + "Andrew Horton", # v0.2 # 2017-11-27 # Updated description and added website. + "Andrew Horton", # v0.3 # 2025-08-02 # Added Shopify-specific header detection. ] -version "0.2" -description "Shopify CMS and ecommerce platform." -website "http://shopify.com/" +version "0.3" +description "Shopify is an e-commerce platform that allows individuals and businesses to create online stores." +website "https://www.shopify.com/" # Matches # matches [ @@ -28,6 +29,13 @@ matches [ { :search => "headers[set-cookie]", :regexp => /cart_sig/, :name=>"cart_sig cookie" }, { :search => "headers[set-cookie]", :regexp => /customer_sig/, :name=>"customer_sig cookie" }, { :search => "headers[set-cookie]", :regexp => /secure_customer_sig/, :name=>"secure_customer_sig cookie" }, + + # Shopify-specific headers + { :search => "headers[shopify-edge-ip]", :name => "Shopify Edge IP header" }, + { :search => "headers[x-shopify-stage]", :name => "Shopify Stage Environment" }, + { :search => "headers[x-sorting-hat-podid]", :name => "Shopify Pod ID" }, + { :search => "headers[x-shopify-api-version]", :name => "Shopify API Version" }, + { :regexp => / "Shopify Digital Wallet Meta Tag" }, ] diff --git a/plugins/tengine-web-server.rb b/plugins/tengine-web-server.rb index 9e2ad5a8..ec8e39dd 100644 --- a/plugins/tengine-web-server.rb +++ b/plugins/tengine-web-server.rb @@ -8,8 +8,9 @@ Plugin.define do name "Tengine-Web-Server" authors [ "Brendan Coles ", # 2012-05-17 + "Andrew Horton", # v0.2 # 2025-08-02 # Added Alibaba-specific Tengine header detection ] -version "0.1" +version "0.2" description "Tengine is a web server originated by Taobao, the largest e-commerce website in Asia. It is based on the popular Nginx HTTP server." website "http://tengine.taobao.org/" @@ -27,6 +28,11 @@ matches [ # HTTP Server Header # Version Detection { :search=>"headers[server]", :version=>/^Tengine\/([^\s]+)/ }, +# Alibaba-specific headers often found with Tengine +{ :search=>"headers[eagleid]", :name=>"Alibaba EagleId Header" }, +{ :search=>"headers[eagleeye-traceid]", :name=>"Alibaba EagleEye Trace ID" }, +{ :search=>"headers[timing-allow-origin]", :name=>"Tengine Timing Allow Origin" }, + ] end diff --git a/plugins/vkontakte-platform.rb b/plugins/vkontakte-platform.rb new file mode 100644 index 00000000..49cdf290 --- /dev/null +++ b/plugins/vkontakte-platform.rb @@ -0,0 +1,25 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "VKontakte-Platform" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "VKontakte (VK) infrastructure identified by server type and specific elements." +website "https://vk.com/" + +matches [ + # Server signature + { :search=>"headers[server]", :regexp=>/^kittenx/i, :name=>"VK Kittenx Server" }, + + # VK API and elements + { :regexp=>/]*src="\/\/vk\.com\/js\/api\/openapi\.js/i, :name=>"VK API Script" }, + { :text=>"", :name=>"VK OG Meta Tag" } +] + +end diff --git a/plugins/weibo-platform.rb b/plugins/weibo-platform.rb new file mode 100644 index 00000000..c4530fdf --- /dev/null +++ b/plugins/weibo-platform.rb @@ -0,0 +1,23 @@ +## +# This file is part of WhatWeb and may be subject to +# redistribution and commercial restrictions. Please see the WhatWeb +# web site for more information on licensing and terms of use. +# https://morningstarsecurity.com/research/whatweb +## +Plugin.define do +name "Weibo-Platform" +authors [ + "Andrew Horton", # v0.1 # 2025-08-02 # Initial version +] +version "0.1" +description "Sina Weibo infrastructure and services identified by unique headers and prefetch links." +website "https://weibo.com/" + +matches [ + # Weibo-specific headers + { :search=>"headers[wbpctrace]", :name=>"Weibo Trace Header" }, + { :text=>"", :name=>"Weibo DNS Prefetch" }, + { :regexp=>/]*content="[^"]*微博/i, :name=>"Weibo OG Meta Tag" } +] + +end diff --git a/whatweb b/whatweb index 20182f15..4d66e2b6 100755 --- a/whatweb +++ b/whatweb @@ -15,7 +15,7 @@ # # Homepage: https://morningstarsecurity.com/research/whatweb # -# Copyright 2009 to 2020 Andrew Horton and Brendan Coles +# Copyright 2009 to 2025 Andrew Horton and Brendan Coles # # This file is part of WhatWeb. #