mirror of
https://github.com/threatexpress/domainhunter
synced 2026-06-08 17:46:25 +00:00
Fix Site Review, Disable MxToolbox, Add -q/quiet option (#31)
* Fix support for BlueCoat reputation checking * Fix Symantec Site Review (Bluecoat) reputation checking + Temporary fix for broken malware domains link. This service is no longer offered in the form used by DomainHunter. + Add internal code comments for readability + Add check for ExpiredDomains username before asking for a password * Add -q/quiet option and minor cleanup * Fix linting error, undefined variable
This commit is contained in:
Andrew Chiles
@@ -4,9 +4,16 @@ Authors Joe Vest (@joevest) & Andrew Chiles (@andrewchiles)
|
||||
|
||||
Domain name selection is an important aspect of preparation for penetration tests and especially Red Team engagements. Commonly, domains that were used previously for benign purposes and were properly categorized can be purchased for only a few dollars. Such domains can allow a team to bypass reputation based web filters and network egress restrictions for phishing and C2 related tasks.
|
||||
|
||||
This Python based tool was written to quickly query the Expireddomains.net search engine for expired/available domains with a previous history of use. It then optionally queries for domain reputation against services like Symantec WebPulse (BlueCoat), IBM X-Force, and Cisco Talos. The primary tool output is a timestamped HTML table style report.
|
||||
This Python based tool was written to quickly query the Expireddomains.net search engine for expired/available domains with a previous history of use. It then optionally queries for domain reputation against services like Symantec Site Review (BlueCoat), IBM X-Force, and Cisco Talos. The primary tool output is a timestamped HTML table style report.
|
||||
|
||||
## Changes
|
||||
## Changelog
|
||||
|
||||
- 07 January 2021
|
||||
+ Fix Symantec Site Review (Bluecoat) reputation checking to bypass XSRF and additional POST parameter checks
|
||||
+ Temporary fix for broken malware domains link. This service is no longer offered in the form used by DomainHunter.
|
||||
+ Add internal code comments for readability
|
||||
+ Add check for ExpiredDomains username before asking for a password
|
||||
+ Disable Google Safe Browsing/PhishTank reputation from MxToolbox as this service has changed
|
||||
|
||||
- 21 February 2020
|
||||
+ updated Pillow version to support Python3.7+
|
||||
|
||||
+80
-35
@@ -18,13 +18,14 @@ import os
|
||||
import sys
|
||||
from urllib.parse import urlparse
|
||||
import getpass
|
||||
import uuid
|
||||
|
||||
__version__ = "20190716"
|
||||
__version__ = "20210107"
|
||||
|
||||
## Functions
|
||||
|
||||
def doSleep(timing):
|
||||
"""Add nmap like random sleep interval for multiple requests"""
|
||||
|
||||
if timing == 0:
|
||||
time.sleep(random.randrange(90,120))
|
||||
elif timing == 1:
|
||||
@@ -38,6 +39,8 @@ def doSleep(timing):
|
||||
# There's no elif timing == 5 here because we don't want to sleep for -t 5
|
||||
|
||||
def checkUmbrella(domain):
|
||||
"""Umbrella Domain reputation service"""
|
||||
|
||||
try:
|
||||
url = 'https://investigate.api.umbrella.com/domains/categorization/?showLabels'
|
||||
postData = [domain]
|
||||
@@ -61,13 +64,41 @@ def checkUmbrella(domain):
|
||||
print('[-] Error retrieving Umbrella reputation! {0}'.format(e))
|
||||
return "error"
|
||||
|
||||
|
||||
def checkBluecoat(domain):
|
||||
try:
|
||||
url = 'https://sitereview.bluecoat.com/resource/lookup'
|
||||
postData = {'url':domain,'captcha':''}
|
||||
"""Symantec Sitereview Domain Reputation"""
|
||||
|
||||
token = str(uuid.uuid4())
|
||||
try:
|
||||
headers = {
|
||||
'User-Agent':useragent,
|
||||
'Referer':'http://sitereview.bluecoat.com/'}
|
||||
|
||||
# Establish our session information
|
||||
response = s.get("https://sitereview.bluecoat.com/",headers=headers,verify=False,proxies=proxies)
|
||||
response = s.head("https://sitereview.bluecoat.com/resource/captcha-request",headers=headers,verify=False,proxies=proxies)
|
||||
|
||||
# Pull the XSRF Token from the cookie jar
|
||||
session_cookies = s.cookies.get_dict()
|
||||
if "XSRF-TOKEN" in session_cookies:
|
||||
token = session_cookies["XSRF-TOKEN"]
|
||||
else:
|
||||
raise NameError("No XSRF-TOKEN found in the cookie jar")
|
||||
|
||||
# Perform SiteReview lookup
|
||||
|
||||
# BlueCoat Added base64 encoded phrases selected at random and sha256 hashing of the JSESSIONID
|
||||
phrases = [
|
||||
'UGxlYXNlIGRvbid0IGZvcmNlIHVzIHRvIHRha2UgbWVhc3VyZXMgdGhhdCB3aWxsIG1ha2UgaXQgbW9yZSBkaWZmaWN1bHQgZm9yIGxlZ2l0aW1hdGUgdXNlcnMgdG8gbGV2ZXJhZ2UgdGhpcyBzZXJ2aWNlLg==',
|
||||
'SWYgeW91IGNhbiByZWFkIHRoaXMsIHlvdSBhcmUgbGlrZWx5IGFib3V0IHRvIGRvIHNvbWV0aGluZyB0aGF0IGlzIGFnYWluc3Qgb3VyIFRlcm1zIG9mIFNlcnZpY2U=',
|
||||
'RXZlbiBpZiB5b3UgYXJlIG5vdCBwYXJ0IG9mIGEgY29tbWVyY2lhbCBvcmdhbml6YXRpb24sIHNjcmlwdGluZyBhZ2FpbnN0IFNpdGUgUmV2aWV3IGlzIHN0aWxsIGFnYWluc3QgdGhlIFRlcm1zIG9mIFNlcnZpY2U=',
|
||||
'U2NyaXB0aW5nIGFnYWluc3QgU2l0ZSBSZXZpZXcgaXMgYWdhaW5zdCB0aGUgU2l0ZSBSZXZpZXcgVGVybXMgb2YgU2VydmljZQ=='
|
||||
]
|
||||
|
||||
postData = {
|
||||
'url':domain,
|
||||
'captcha':'',
|
||||
'key':'%032x' % random.getrandbits(256), # Generate a random 256bit "hash-like" string
|
||||
'phrase':random.choice(phrases), # Pick a random base64 phrase from the list
|
||||
'source':'new-lookup'}
|
||||
|
||||
headers = {'User-Agent':useragent,
|
||||
'Accept':'application/json, text/plain, */*',
|
||||
@@ -76,14 +107,13 @@ def checkBluecoat(domain):
|
||||
'X-XSRF-TOKEN':token,
|
||||
'Referer':'http://sitereview.bluecoat.com/'}
|
||||
|
||||
c = {
|
||||
"JSESSIONID":str(uuid.uuid4()).upper().replace("-", ""),
|
||||
"XSRF-TOKEN":token
|
||||
}
|
||||
|
||||
print('[*] BlueCoat: {}'.format(domain))
|
||||
response = s.post('https://sitereview.bluecoat.com/resource/lookup',headers=headers,json=postData,verify=False,proxies=proxies)
|
||||
|
||||
response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
|
||||
# Check for any HTTP errors
|
||||
if response.status_code != 200:
|
||||
a = "HTTP Error ({}-{}) - Is your IP blocked?".format(response.status_code,response.reason)
|
||||
else:
|
||||
responseJSON = json.loads(response.text)
|
||||
|
||||
if 'errorType' in responseJSON:
|
||||
@@ -103,13 +133,13 @@ def checkBluecoat(domain):
|
||||
if captcha:
|
||||
b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
|
||||
|
||||
# Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
|
||||
# Send CAPTCHA solution via GET since inclusion with the domain categorization request doesn't work anymore
|
||||
captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
|
||||
print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
|
||||
response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
|
||||
|
||||
# Try the categorization request again
|
||||
response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
|
||||
response = s.post(url,headers=headers,json=postData,verify=False,proxies=proxies)
|
||||
|
||||
responseJSON = json.loads(response.text)
|
||||
|
||||
@@ -121,7 +151,6 @@ def checkBluecoat(domain):
|
||||
print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
|
||||
else:
|
||||
print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
|
||||
|
||||
return a
|
||||
|
||||
except Exception as e:
|
||||
@@ -129,6 +158,8 @@ def checkBluecoat(domain):
|
||||
return "error"
|
||||
|
||||
def checkIBMXForce(domain):
|
||||
"""IBM XForce Domain Reputation"""
|
||||
|
||||
try:
|
||||
url = 'https://exchange.xforce.ibmcloud.com/url/{}'.format(domain)
|
||||
headers = {'User-Agent':useragent,
|
||||
@@ -155,7 +186,7 @@ def checkIBMXForce(domain):
|
||||
else:
|
||||
categories = ''
|
||||
# Parse all dictionary keys and append to single string to get Category names
|
||||
for key in responseJSON["result"]['cats']:
|
||||
for key in responseJSON['result']['cats']:
|
||||
categories += '{0}, '.format(str(key))
|
||||
|
||||
a = '{0}(Score: {1})'.format(categories,str(responseJSON['result']['score']))
|
||||
@@ -167,6 +198,8 @@ def checkIBMXForce(domain):
|
||||
return "error"
|
||||
|
||||
def checkTalos(domain):
|
||||
"""Cisco Talos Domain Reputation"""
|
||||
|
||||
url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
|
||||
headers = {'User-Agent':useragent,
|
||||
'Referer':url}
|
||||
@@ -195,6 +228,7 @@ def checkTalos(domain):
|
||||
return "error"
|
||||
|
||||
def checkMXToolbox(domain):
|
||||
""" Checks the MXToolbox service for Google SafeBrowsing and PhishTank information. Currently broken"""
|
||||
url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
|
||||
headers = {'User-Agent':useragent,
|
||||
'Origin':url,
|
||||
@@ -254,6 +288,8 @@ def checkMXToolbox(domain):
|
||||
return "error"
|
||||
|
||||
def downloadMalwareDomains(malwaredomainsURL):
|
||||
"""Downloads a current list of known malicious domains"""
|
||||
|
||||
url = malwaredomainsURL
|
||||
response = s.get(url=url,headers=headers,verify=False,proxies=proxies)
|
||||
responseText = response.text
|
||||
@@ -263,6 +299,8 @@ def downloadMalwareDomains(malwaredomainsURL):
|
||||
print("[-] Error reaching:{} Status: {}").format(url, response.status_code)
|
||||
|
||||
def checkDomain(domain):
|
||||
"""Executes various domain reputation checks included in the project"""
|
||||
|
||||
print('[*] Fetching domain reputation for: {}'.format(domain))
|
||||
|
||||
if domain in maldomainsList:
|
||||
@@ -277,8 +315,10 @@ def checkDomain(domain):
|
||||
ciscotalos = checkTalos(domain)
|
||||
print("[+] {}: {}".format(domain, ciscotalos))
|
||||
|
||||
mxtoolbox = checkMXToolbox(domain)
|
||||
print("[+] {}: {}".format(domain, mxtoolbox))
|
||||
#This service has completely changed, removing for now
|
||||
#mxtoolbox = checkMXToolbox(domain)
|
||||
#print("[+] {}: {}".format(domain, mxtoolbox))
|
||||
mxtoolbox = "-"
|
||||
|
||||
umbrella = "not available"
|
||||
if len(umbrella_apikey):
|
||||
@@ -291,8 +331,9 @@ def checkDomain(domain):
|
||||
return results
|
||||
|
||||
def solveCaptcha(url,session):
|
||||
# Downloads CAPTCHA image and saves to current directory for OCR with tesseract
|
||||
# Returns CAPTCHA string or False if error occured
|
||||
"""Downloads CAPTCHA image and saves to current directory for OCR with tesseract
|
||||
Returns CAPTCHA string or False if error occured
|
||||
"""
|
||||
|
||||
jpeg = 'captcha.jpg'
|
||||
|
||||
@@ -324,7 +365,7 @@ def solveCaptcha(url,session):
|
||||
return False
|
||||
|
||||
def drawTable(header,data):
|
||||
|
||||
"""Generates a text based table for printing to the console"""
|
||||
data.insert(0,header)
|
||||
t = Texttable(max_width=maxwidth)
|
||||
t.add_rows(data)
|
||||
@@ -333,6 +374,8 @@ def drawTable(header,data):
|
||||
return(t.draw())
|
||||
|
||||
def loginExpiredDomains():
|
||||
"""Login to the ExpiredDomains site with supplied credentials"""
|
||||
|
||||
data = "login=%s&password=%s&redirect_2_url=/" % (username, password)
|
||||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
r = s.post(expireddomainHost + "/login/", headers=headers, data=data, proxies=proxies, verify=False, allow_redirects=False)
|
||||
@@ -387,6 +430,7 @@ Examples:
|
||||
parser.add_argument('-ks','--keyword-start', help='Keyword starts with used to refine search results', required=False, default="", type=str, dest='keyword_start')
|
||||
parser.add_argument('-ke','--keyword-end', help='Keyword ends with used to refine search results', required=False, default="", type=str, dest='keyword_end')
|
||||
parser.add_argument('-um','--umbrella-apikey', help='API Key for umbrella (paid)', required=False, default="", type=str, dest='umbrella_apikey')
|
||||
parser.add_argument('-q','--quiet', help='Surpress initial ASCII art and header', required=False, default=False, action='store_true', dest='quiet')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load dependent modules
|
||||
@@ -448,8 +492,7 @@ Examples:
|
||||
|
||||
umbrella_apikey = args.umbrella_apikey
|
||||
|
||||
malwaredomainsURL = 'http://mirror1.malwaredomains.com/files/justdomains'
|
||||
|
||||
malwaredomainsURL = 'https://gitlab.com/gerowen/old-malware-domains-ad-list/-/raw/master/malwaredomainslist.txt'
|
||||
expireddomainsqueryURL = 'https://www.expireddomains.net/domain-name-search'
|
||||
expireddomainHost = "https://member.expireddomains.net"
|
||||
|
||||
@@ -470,7 +513,7 @@ Examples:
|
||||
proxy_parts = urlparse(args.proxy)
|
||||
proxies["http"] = "http://%s" % (proxy_parts.netloc)
|
||||
proxies["https"] = "https://%s" % (proxy_parts.netloc)
|
||||
|
||||
s.proxies = proxies
|
||||
title = '''
|
||||
____ ___ __ __ _ ___ _ _ _ _ _ _ _ _ _____ _____ ____
|
||||
| _ \ / _ \| \/ | / \ |_ _| \ | | | | | | | | | \ | |_ _| ____| _ \
|
||||
@@ -478,19 +521,19 @@ Examples:
|
||||
| |_| | |_| | | | |/ ___ \ | || |\ | | _ | |_| | |\ | | | | |___| _ <
|
||||
|____/ \___/|_| |_/_/ \_\___|_| \_| |_| |_|\___/|_| \_| |_| |_____|_| \_\ '''
|
||||
|
||||
# Print header
|
||||
if not (args.quiet):
|
||||
print(title)
|
||||
print("")
|
||||
print("Expired Domains Reputation Checker")
|
||||
print("Authors: @joevest and @andrewchiles\n")
|
||||
print("DISCLAIMER: This is for educational purposes only!")
|
||||
disclaimer = '''It is designed to promote education and the improvement of computer/cyber security.
|
||||
print('''\nExpired Domains Reputation Checker
|
||||
Authors: @joevest and @andrewchiles\n
|
||||
DISCLAIMER: This is for educational purposes only!
|
||||
It is designed to promote education and the improvement of computer/cyber security.
|
||||
The authors or employers are not liable for any illegal act or misuse performed by any user of this tool.
|
||||
If you plan to use this content for illegal purpose, don't. Have a nice day :)'''
|
||||
print(disclaimer)
|
||||
print("")
|
||||
If you plan to use this content for illegal purpose, don't. Have a nice day :)\n''')
|
||||
|
||||
# Download known malware domains
|
||||
print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
|
||||
# print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
|
||||
|
||||
maldomains = downloadMalwareDomains(malwaredomainsURL)
|
||||
maldomainsList = maldomains.split("\n")
|
||||
|
||||
@@ -527,7 +570,9 @@ If you plan to use this content for illegal purpose, don't. Have a nice day :)'
|
||||
|
||||
# Generate list of URLs to query for expired/deleted domains
|
||||
urls = []
|
||||
|
||||
if username == None or username == "":
|
||||
print('[-] Error: ExpiredDomains.net requires a username! Use the --username parameter')
|
||||
exit(1)
|
||||
if args.password == None or args.password == "":
|
||||
password = getpass.getpass("expireddomains.net Password: ")
|
||||
|
||||
|
||||
@@ -5,4 +5,3 @@ lxml
|
||||
pillow
|
||||
pytesseract
|
||||
urllib3
|
||||
uuid
|
||||
|
||||
Reference in New Issue
Block a user