diff --git a/playbooks/UrlScan_IO_Dynamic_Analysis.json b/playbooks/UrlScan_IO_Dynamic_Analysis.json new file mode 100644 index 0000000000..259106649b --- /dev/null +++ b/playbooks/UrlScan_IO_Dynamic_Analysis.json @@ -0,0 +1,558 @@ +{ + "blockly": false, + "blockly_xml": "", + "category": "Dynamic Analysis", + "coa": { + "data": { + "description": "Accepts a URL for detonation analysis on the objects. Generates a global report and a per observable sub-report and normalized score. The score can be customized based on a variety of factors.\n\nRef: https://d3fend.mitre.org/technique/d3f:IdentifierReputationAnalysis/", + "edges": [ + { + "id": "port_0_to_port_2", + "sourceNode": "0", + "sourcePort": "0_out", + "targetNode": "2", + "targetPort": "2_in" + }, + { + "conditions": [ + { + "index": 0 + } + ], + "id": "port_2_to_port_3", + "sourceNode": "2", + "sourcePort": "2_out", + "targetNode": "3", + "targetPort": "3_in" + }, + { + "id": "port_3_to_port_4", + "sourceNode": "3", + "sourcePort": "3_out", + "targetNode": "4", + "targetPort": "4_in" + }, + { + "id": "port_5_to_port_6", + "sourceNode": "5", + "sourcePort": "5_out", + "targetNode": "6", + "targetPort": "6_in" + }, + { + "conditions": [ + { + "index": 0 + } + ], + "id": "port_4_to_port_7", + "sourceNode": "4", + "sourcePort": "4_out", + "targetNode": "7", + "targetPort": "7_in" + }, + { + "conditions": [ + { + "index": 1 + } + ], + "id": "port_7_to_port_8", + "sourceNode": "7", + "sourcePort": "7_out", + "targetNode": "8", + "targetPort": "8_in" + }, + { + "id": "port_8_to_port_9", + "sourceNode": "8", + "sourcePort": "8_out", + "targetNode": "9", + "targetPort": "9_in" + }, + { + "id": "port_11_to_port_1", + "sourceNode": "11", + "sourcePort": "11_out", + "targetNode": "1", + "targetPort": "1_in" + }, + { + "id": "port_10_to_port_1", + "sourceNode": "10", + "sourcePort": "10_out", + "targetNode": "1", + "targetPort": "1_in" + }, + { + "id": "port_6_to_port_10", + "sourceNode": "6", + "sourcePort": "6_out", + "targetNode": "10", + "targetPort": "10_in" + }, + { + "id": "port_9_to_port_11", + "sourceNode": "9", + "sourcePort": "9_out", + "targetNode": "11", + "targetPort": "11_in" + }, + { + "conditions": [ + { + "index": 0 + } + ], + "id": "port_7_to_port_5", + "sourceNode": "7", + "sourcePort": "7_out", + "targetNode": "5", + "targetPort": "5_in" + } + ], + "hash": "662e0c1ead292892eb76490c42919b482032f0a2", + "nodes": { + "0": { + "data": { + "advanced": { + "join": [] + }, + "functionName": "on_start", + "id": "0", + "type": "start" + }, + "errors": {}, + "id": "0", + "type": "start", + "warnings": {}, + "x": 190, + "y": -1.2789769243681803e-13 + }, + "1": { + "data": { + "advanced": { + "join": [] + }, + "functionName": "on_finish", + "id": "1", + "type": "end" + }, + "errors": {}, + "id": "1", + "type": "end", + "userCode": "\n # Write your custom code here...\n no_error_code_format_report_url = phantom.get_format_data(name=\"no_error_code_format_report_url\")\n error_code_format_report_url = phantom.get_format_data(name=\"error_code_format_report_url\")\n markdown_report_combined_value = phantom.concatenate(no_error_code_format_report_url, error_code_format_report_url)\n output['markdown_report'] = markdown_report_combined_value\n", + "warnings": {}, + "x": 200, + "y": 1440 + }, + "10": { + "data": { + "advanced": { + "customName": "build url output with error code", + "customNameId": 0, + "description": "This block uses custom code to generate an observable dictionary to output into the observables data path.", + "join": [], + "note": "This block uses custom code to generate an observable dictionary to output into the observables data path." + }, + "functionId": 3, + "functionName": "build_url_output_with_error_code", + "id": "10", + "inputParameters": [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_error_code:custom_function:url_score_object", + "url_reputation:action_result.data.*.task.reportURL" + ], + "outputVariables": [ + "observable_array" + ], + "type": "code" + }, + "errors": {}, + "id": "10", + "type": "code", + "userCode": "\n # Write your custom code here...\n # Write your custom code here...\n from urllib.parse import urlparse\n build_url_output_with_error_code__observable_array = []\n\n # Build URL\n url_scan_io_task_reporturl = [str(i or 'no report url') for i in url_reputation_result_item_1]\n url_scan_io_parameter_url = [str(i or '') for i in url_reputation_parameter_url]\n url_scan_io_url_score_object = normalize_score_url_with_error_code__url_score_object\n \n phantom.debug(\"url_reputation_parameter_url: {}\".format(url_reputation_parameter_url))\n #phantom.debug(\"url_reputation_result_item_1: {}\".format(url_reputation_result_item_1))\n #phantom.debug(\"normalize_score_url_with_error_code__url_score_object: {}\".format(normalize_score_url_with_error_code__url_score_object))\n for url, external_id, url_object in zip(url_scan_io_parameter_url, url_scan_io_task_reporturl, url_scan_io_url_score_object):\n parsed_url = urlparse(url)\n phantom.debug(\"{} {} {} parsed_url: {}\".format(url, external_id, url_object, parsed_url))\n observable_object = {\n \"value\": url,\n \"type\": \"url\",\n \"sandbox\": {\n \"score_id\": url_object['score_id'],\n \"score\": url_object['score'],\n \"confidence\": url_object['confidence']\n }, \n \"attributes\": {\n \"hostname\": parsed_url.hostname,\n \"scheme\": parsed_url.scheme\n },\n \"categories\": url_object['categories'],\n \"description\" :url_object['description'],\n \"source\": \"urlscan.io\",\n \"source_link\": f\"{external_id}\"\n }\n\n if parsed_url.path:\n observable_object['attributes']['path'] = parsed_url.path\n if parsed_url.query:\n observable_object['attributes']['query'] = parsed_url.query\n if parsed_url.port:\n observable_object['attributes']['port'] = parsed_url.port\n\n \n build_url_output_with_error_code__observable_array.append(observable_object)\n phantom.debug(\"build_url_output_with_error_code__observable_array: {}\".format(build_url_output_with_error_code__observable_array))\n", + "warnings": {}, + "x": 340, + "y": 1260 + }, + "11": { + "data": { + "advanced": { + "customName": "build url output with no error code", + "customNameId": 0, + "description": "This block uses custom code to generate an observable dictionary to output into the observables data path.", + "join": [], + "note": "This block uses custom code to generate an observable dictionary to output into the observables data path." + }, + "functionId": 4, + "functionName": "build_url_output_with_no_error_code", + "id": "11", + "inputParameters": [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_no_error_code:custom_function:url_score_object", + "url_reputation:action_result.data.*.task.reportURL" + ], + "outputVariables": [ + "observable_array" + ], + "type": "code" + }, + "errors": {}, + "id": "11", + "type": "code", + "userCode": "\n # Write your custom code here...\n from urllib.parse import urlparse\n build_url_output_with_no_error_code__observable_array = []\n\n # Build URL\n phantom.debug(url_reputation_parameter_url)\n for url, external_id, url_object in zip(url_reputation_parameter_url, url_reputation_result_item_1, normalize_score_url_with_no_error_code__url_score_object):\n parsed_url = urlparse(url)\n phantom.debug(\"{} {} {} parsed_url: {}\".format(url, external_id, url_object, parsed_url))\n observable_object = {\n \"value\": url,\n \"type\": \"url\",\n \"sandbox\": {\n \"score_id\": url_object['score_id'],\n \"score\": url_object['score'],\n \"confidence\": url_object['confidence']\n },\n \"attributes\": {\n \"hostname\": parsed_url.hostname,\n \"scheme\": parsed_url.scheme\n },\n \"categories\": url_object['categories'],\n \"description\" :url_object['description'],\n \"source\": \"urlscan.io\",\n \"source_link\": f\"{external_id}\"\n }\n if parsed_url.path:\n observable_object['attributes']['path'] = parsed_url.path\n if parsed_url.query:\n observable_object['attributes']['query'] = parsed_url.query\n if parsed_url.port:\n observable_object['attributes']['port'] = parsed_url.port\n \n build_url_output_with_no_error_code__observable_array.append(observable_object)\n phantom.debug(\"build_url_output_with_no_error_code__observable_array: {}\".format(build_url_output_with_no_error_code__observable_array))\n", + "warnings": {}, + "x": 0, + "y": 1260 + }, + "2": { + "data": { + "advanced": { + "customName": "url input filter", + "customNameId": 0, + "description": "Determine branches based on provided inputs.", + "join": [], + "note": "Determine branches based on provided inputs." + }, + "conditions": [ + { + "comparisons": [ + { + "conditionIndex": 0, + "op": "!=", + "param": "playbook_input:url", + "value": "" + } + ], + "conditionIndex": 0, + "customName": "url input filter", + "logic": "and" + } + ], + "functionId": 1, + "functionName": "url_input_filter", + "id": "2", + "type": "filter" + }, + "errors": {}, + "id": "2", + "type": "filter", + "warnings": {}, + "x": 230, + "y": 148 + }, + "3": { + "data": { + "action": "detonate url", + "actionType": "investigate", + "advanced": { + "customName": "url reputation", + "customNameId": 0, + "description": "Queries urlscan.io for information about the provided URL(s)", + "join": [], + "note": "Queries urlscan.io for information about the provided URL(s)" + }, + "connector": "urlscan.io", + "connectorConfigs": [ + "urlscan.io" + ], + "connectorId": "c46c00cd-7231-4dd3-8d8e-02b9fa0e14a2", + "connectorVersion": "v1", + "functionId": 1, + "functionName": "url_reputation", + "id": "3", + "parameters": { + "get_result": true, + "private": true, + "url": "playbook_input:url" + }, + "requiredParameters": [ + { + "data_type": "string", + "field": "url" + }, + { + "data_type": "boolean", + "default": true, + "field": "private" + }, + { + "data_type": "boolean", + "default": true, + "field": "get_result" + } + ], + "type": "action" + }, + "errors": {}, + "id": "3", + "type": "action", + "warnings": {}, + "x": 170, + "y": 328 + }, + "4": { + "data": { + "advanced": { + "customName": "urlscanio summary filter", + "customNameId": 0, + "description": "Filters successful url reputation results.", + "join": [], + "note": "Filters successful url reputation results." + }, + "conditions": [ + { + "comparisons": [ + { + "conditionIndex": 0, + "op": "==", + "param": "url_reputation:action_result.status", + "value": "success" + } + ], + "conditionIndex": 0, + "customName": "success_urlscan_query", + "logic": "and" + } + ], + "functionId": 2, + "functionName": "urlscanio_summary_filter", + "id": "4", + "type": "filter" + }, + "errors": {}, + "id": "4", + "type": "filter", + "warnings": {}, + "x": 240, + "y": 500 + }, + "5": { + "customCode": null, + "data": { + "advanced": { + "customName": "normalize score url with error code", + "customNameId": 0, + "description": "This block uses custom code for normalizing score. Adjust the logic as desired in the documented sections.", + "join": [], + "note": "This block uses custom code for normalizing score. Adjust the logic as desired in the documented sections." + }, + "functionId": 1, + "functionName": "normalize_score_url_with_error_code", + "id": "5", + "inputParameters": [ + "url_reputation:action_result.parameter.url", + "url_reputation:action_result.data.*.message", + "url_reputation:action_result.data.*.status", + "url_reputation:action_result.data.*.description" + ], + "outputVariables": [ + "url_score_object", + "scores", + "categories", + "confidence" + ], + "type": "code" + }, + "errors": {}, + "id": "5", + "type": "code", + "userCode": "\n # Write your custom code here...\n url_reputation_data_message = [str(i or '') for i in url_reputation_result_item_1] \n url_reputation_data_status = [str(i or '') for i in url_reputation_result_item_2] \n url_reputation_data_description = [str(i or '') for i in url_reputation_result_item_3] \n\n \n normalize_score_url_with_error_code__url_score_object = []\n normalize_score_url_with_error_code__scores = []\n normalize_score_url_with_error_code__categories = []\n\n \n #phantom.debug(\"url_reputation_parameter_url: {}\".format(url_reputation_parameter_url))\n #phantom.debug(\"url_reputation_data_message: {}\".format(url_reputation_data_message))\n #phantom.debug(\"url_reputation_data_status: {}\".format(url_reputation_data_status))\n #phantom.debug(\"url_reputation_data_description: {}\".format(url_reputation_data_description))\n\n \n\n \n urlscan_score_table = {\n \"0\":\"Legitimate\",\n \"1\":\"Very_Safe\",\n \"2\":\"Safe\",\n \"3\":\"Probably_Safe\",\n \"4\":\"Leans_Safe\",\n \"5\":\"May_not_be_Safe\",\n \"6\":\"Exercise_Caution\",\n \"7\":\"Suspicious_or_Risky\",\n \"8\":\"Possibly_Malicious\",\n \"9\":\"Probably_Malicious\",\n \"10\":\"Malicious\",\n \"error_code_query\" : \"error code return, check the error code descriptions\"\n\n }\n \n url_scan_io_error_code ={\n \"blacklist\" : \"Blacklisted URL or Domain\",\n \"spam\" : \"Spammy URL or Domain\",\n \"invalid_hostname\" : \"Invalid Hostname URL or Domain\",\n \"missing_url\" : \"Missing URL OR Domain Property\",\n \"auth\" : \"HTTP basic auth information\",\n \"not_be_resolved\" : \"Non-resolvable hostname (A, AAAA, CNAME)\"\n }\n ## URLSCAN.io return error code especially if the url or domain was already in their blacklist database. \n ## below are the common error code message base on their\n ## - \"Blacklisted domains and URLs\" : requested to be blacklisted by their respective owners.\n ## - \"Spammy submissions\" : of URLs known to be used only for spamming this service.\n ## - \"Invalid hostnames\" : or invalid protocol schemes (FTP etc).\n ## - \"Missing URL property\" : ... yes, it does happen.\n ## - \"Contains HTTP basic auth information\" : ... yes, that happens as well.\n ## - \"Non-resolvable hostnames (A, AAAA, CNAME)\" : which we will not even try to scan.\n \n blank_result = \"--\"\n category = \"\"\n score = \"\"\n error_message = \"\"\n \n ## check if there is error code return upon URL reputation query\n \n for url_descp in url_reputation_data_description:\n \n for key, value in url_scan_io_error_code.items():\n if key.replace(\"_\",\" \").lower() in url_descp.lower():\n error_message = url_scan_io_error_code[key]\n \n # Attach final object\n normalize_score_url_with_error_code__categories.append(error_message) \n normalize_score_url_with_error_code__url_score_object.append({'score': urlscan_score_table['error_code_query'], 'confidence':\"\", 'score_id': \"\", \"malicious_tag_verdicts\": \"\", 'categories': error_message, \"description\": url_descp})\n normalize_score_url_with_error_code__scores.append(urlscan_score_table['error_code_query'])\n \n phantom.debug(\"normalize_score_url_with_error_code__url_score_object: {}\".format(normalize_score_url_with_error_code__url_score_object))\n phantom.debug(\"normalize_score_url_with_error_code__scores: {}\".format(normalize_score_url_with_error_code__scores))\n phantom.debug(\"normalize_score_url_with_error_code__categories: {}\".format(normalize_score_url_with_error_code__categories))\n \n \n", + "warnings": {}, + "x": 340, + "y": 860 + }, + "6": { + "data": { + "advanced": { + "customName": "error code format report url", + "customNameId": 0, + "description": "Format a summary table with the information gathered from the playbook.", + "join": [], + "note": "Format a summary table with the information gathered from the playbook." + }, + "functionId": 1, + "functionName": "error_code_format_report_url", + "id": "6", + "parameters": [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_error_code:custom_function:confidence", + "normalize_score_url_with_error_code:custom_function:scores", + "normalize_score_url_with_error_code:custom_function:categories", + "url_reputation:action_result.data.*.task.reportURL" + ], + "template": "SOAR analyzed URL(s) using urlscan.io. The table below shows a summary of the information gathered.\n\n| URL | Normalized Score | Confidence |Categories | Report Link | Source |\n| --- | --- | --- | --- | --- |\n%%\n| `{0}` | {1} | {2} | {3} | {4} | urlscan.io |\n\n", + "type": "format" + }, + "errors": {}, + "id": "6", + "type": "format", + "userCode": "\n # Write your custom code here...\n phantom.debug(phantom.format(container=container, template=template, parameters=parameters, name=\"error_code_format_report_url\"))\n", + "warnings": {}, + "x": 340, + "y": 1060 + }, + "7": { + "data": { + "advanced": { + "customName": "urlscanio error code filter", + "customNameId": 0, + "description": "This filter is designed to avoid nonetype value in some list and dictionary object due to error code return value of urlscan.io especially if the URL or domain is in their blacklist database", + "join": [], + "note": "This filter is designed to avoid nonetype value in some list and dictionary object due to error code return value of urlscan.io especially if the URL or domain is in their blacklist database" + }, + "conditions": [ + { + "comparisons": [ + { + "conditionIndex": 0, + "op": "==", + "param": "filtered-data:urlscanio_summary_filter:condition_1:url_reputation:action_result.data.*.status", + "value": "400" + } + ], + "conditionIndex": 0, + "customName": "error_code", + "logic": "and" + }, + { + "comparisons": [ + { + "conditionIndex": 1, + "op": "!=", + "param": "filtered-data:urlscanio_summary_filter:condition_1:url_reputation:action_result.data.*.status", + "value": "400" + } + ], + "conditionIndex": 1, + "customName": "no_error_code", + "logic": "or" + } + ], + "functionId": 3, + "functionName": "urlscanio_error_code_filter", + "id": "7", + "type": "filter" + }, + "errors": {}, + "id": "7", + "type": "filter", + "warnings": {}, + "x": 240, + "y": 680 + }, + "8": { + "data": { + "advanced": { + "customName": "normalize score url with no error code", + "customNameId": 0, + "description": "This block uses custom code for normalizing score. Adjust the logic as desired in the documented sections.", + "join": [], + "note": "This block uses custom code for normalizing score. Adjust the logic as desired in the documented sections." + }, + "functionId": 2, + "functionName": "normalize_score_url_with_no_error_code", + "id": "8", + "inputParameters": [ + "url_reputation:action_result.data.*.verdicts.overall", + "url_reputation:action_result.data.*.verdicts.urlscan", + "url_reputation:action_result.data.*.verdicts.engines", + "url_reputation:action_result.data.*.verdicts.community" + ], + "outputVariables": [ + "url_score_object", + "scores", + "categories", + "confidence" + ], + "type": "code" + }, + "errors": {}, + "id": "8", + "type": "code", + "userCode": "\n # Write your custom code here...\n \n normalize_score_url_with_no_error_code__url_score_object = []\n normalize_score_url_with_no_error_code__scores = []\n normalize_score_url_with_no_error_code__categories = []\n \n url_reputation_verdicts_overall_dict = [(i or {}) for i in url_reputation_result_item_0] \n url_reputation_verdicts_urlscan_dict = [(i or {}) for i in url_reputation_result_item_1] \n url_reputation_verdicts_engine_dict = [(i or {}) for i in url_reputation_result_item_2] \n url_reputation_verdicts_community_dict = [(i or {}) for i in url_reputation_result_item_3] \n \n #phantom.debug(\"url_reputation_verdicts_overall_dict: {}\".format(url_reputation_verdicts_overall_dict))\n #phantom.debug(\"url_reputation_verdicts_urlscan_dict: {}\".format(url_reputation_verdicts_urlscan_dict))\n #phantom.debug(\"url_reputation_verdicts_engine_dict: {}\".format(url_reputation_verdicts_engine_dict))\n #phantom.debug(\"url_reputation_verdicts_community_dict: {}\".format(url_reputation_verdicts_community_dict))\n \n \n urlscan_score_table = {\n \"0\":\"Legitimate\",\n \"1\":\"Very_Safe\",\n \"2\":\"Safe\",\n \"3\":\"Probably_Safe\",\n \"4\":\"Leans_Safe\",\n \"5\":\"May_not_be_Safe\",\n \"6\":\"Exercise_Caution\",\n \"7\":\"Suspicious_or_Risky\",\n \"8\":\"Possibly_Malicious\",\n \"9\":\"Probably_Malicious\",\n \"10\":\"Malicious\"\n }\n \n ## if there is no error code, urlscan.io will continue to detonate the URL and query scores in several verdicts object\n \n ## Normalize reputation on a -100 (legitimate) to 100 point scale based on number of malicious and suspicious divided by different urlscan.io verdict objects.\n ## This can be adjusted to include whatever logic is desired.\n \n for i in range(0,len(url_reputation_verdicts_overall_dict)):\n if url_reputation_verdicts_overall_dict[i] != {} or url_reputation_verdicts_urlscan_dict[i] != {} or url_reputation_verdicts_engine_dict[i] != {} or url_reputation_verdicts_community_dict[i] != {}:\n summary_score = url_reputation_verdicts_overall_dict[i]['score'] + url_reputation_verdicts_urlscan_dict[i]['score'] + url_reputation_verdicts_engine_dict[i]['score'] + url_reputation_verdicts_community_dict[i]['score']\n summary_malicious_verdicts = url_reputation_verdicts_overall_dict[i]['malicious'] or url_reputation_verdicts_urlscan_dict[i]['malicious'] or url_reputation_verdicts_engine_dict[i]['malicious'] or url_reputation_verdicts_community_dict[i]['malicious']\n summary_of_malicious_tag = int(url_reputation_verdicts_overall_dict[i]['malicious']) + int(url_reputation_verdicts_urlscan_dict[i]['malicious']) + int(url_reputation_verdicts_engine_dict[i]['malicious']) + int(url_reputation_verdicts_community_dict[i]['malicious'])\n summary_categories = url_reputation_verdicts_overall_dict[i]['categories'] + url_reputation_verdicts_urlscan_dict[i]['categories'] + url_reputation_verdicts_engine_dict[i]['categories'] + url_reputation_verdicts_community_dict[i]['categories']\n \n \n ## customized score id calculation\n \n log_result = (summary_score/4) # avg score from different urlscan.io score object (engine_score, overall_verdict_score, urlscan_verdicts_score and community score)\n score_id = int(log_result) \n \n\n if score_id < -50:\n score_id = \"0\"\n elif score_id < 0 and score_id >= -50:\n score_id = \"1\"\n elif score_id >= 0 and score_id <= 10:\n score_id = \"2\"\n elif score_id > 10 and score_id <= 20:\n score_id = \"3\"\n elif score_id > 20 and score_id <= 30:\n score_id = \"4\"\n elif score_id > 30 and score_id <= 40:\n score_id = \"5\"\n elif score_id > 40 and score_id <= 50:\n score_id = \"6\"\n elif score_id > 50 and score_id <= 60:\n score_id = \"7\"\n elif score_id > 70 and score_id <= 80:\n score_id = \"8\"\n elif score_id > 80 and score_id <= 90:\n score_id = \"9\"\n elif score_id > 90 and score_id <= 100:\n score_id = \"10\"\n\n score = urlscan_score_table[str(score_id)]\n\n malicious_tag_stats = (summary_of_malicious_tag, 4)\n\n # Attach final object\n normalize_score_url_with_no_error_code__categories.append(summary_categories) \n normalize_score_url_with_no_error_code__url_score_object.append({'score': score, 'confidence':log_result, 'score_id': score_id, \"malicious_tag_verdicts\": summary_malicious_verdicts, \"malicious_tag_stats\": malicious_tag_stats , 'categories': summary_categories, \"description\": \"\"})\n normalize_score_url_with_no_error_code__scores.append(score)\n \n phantom.debug(\"normalize_score_url_with_no_error_code__categories: {}\".format(normalize_score_url_with_no_error_code__categories))\n phantom.debug(\"normalize_score_url_with_no_error_code__url_score_object: {}\".format(normalize_score_url_with_no_error_code__url_score_object))\n phantom.debug(\"normalize_score_url_with_no_error_code__scores: {}\".format(normalize_score_url_with_no_error_code__scores))\n", + "warnings": {}, + "x": 0, + "y": 860 + }, + "9": { + "data": { + "advanced": { + "customName": "no error code format report url ", + "customNameId": 0, + "description": "Format a summary table with the information gathered from the playbook.", + "join": [], + "note": "Format a summary table with the information gathered from the playbook." + }, + "functionId": 2, + "functionName": "no_error_code_format_report_url", + "id": "9", + "parameters": [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_no_error_code:custom_function:scores", + "normalize_score_url_with_no_error_code:custom_function:confidence", + "normalize_score_url_with_no_error_code:custom_function:categories", + "url_reputation:action_result.data.*.task.reportURL" + ], + "template": "SOAR analyzed URL(s) using urlscan.io. The table below shows a summary of the information gathered.\n\n| URL | Normalized Score |Confidence | Categories | Report Link | Source |\n| --- | --- | --- | --- | --- |\n%%\n| `{0}` | {1} | {2} | {3} |{4} | urlscan.io |\n", + "type": "format" + }, + "errors": {}, + "id": "9", + "type": "format", + "userCode": "\n # Write your custom code here...\n phantom.debug(phantom.format(container=container, template=template, parameters=parameters, name=\"no_error_code_format_report_url\"))\n", + "warnings": {}, + "x": 0, + "y": 1060 + } + }, + "notes": "Inputs: url\nInteractions: UrlScan.io\nActions: url detonation\nOutputs: report, observables" + }, + "input_spec": [ + { + "contains": [ + "url", + "domain", + "ip" + ], + "description": "A URL provided for reputation analysis - urlscan.io", + "name": "url" + } + ], + "output_spec": [ + { + "contains": [], + "datapaths": [ + "build_url_output_with_error_code:custom_function:observable_array", + "build_url_output_with_no_error_code:custom_function:observable_array" + ], + "deduplicate": false, + "description": "An array of observable dictionaries with value, type, score, score_id, and categories.", + "metadata": {}, + "name": "observable" + } + ], + "playbook_type": "data", + "python_version": "3", + "schema": "5.0.8", + "version": "5.5.0.108488" + }, + "create_time": "2023-03-27T12:19:43.255588+00:00", + "draft_mode": false, + "labels": [ + "*" + ], + "tags": [ + "url", + "domain", + "ip", + "sandbox", + "D3-URA", + "D3-DNRA", + "D3-IPRA", + "D3-IRA", + "urlscan.io", + "file_hash" + ] +} diff --git a/playbooks/UrlScan_IO_Dynamic_Analysis.png b/playbooks/UrlScan_IO_Dynamic_Analysis.png new file mode 100644 index 0000000000..eae57860f4 Binary files /dev/null and b/playbooks/UrlScan_IO_Dynamic_Analysis.png differ diff --git a/playbooks/UrlScan_IO_Dynamic_Analysis.py b/playbooks/UrlScan_IO_Dynamic_Analysis.py new file mode 100644 index 0000000000..4c38350830 --- /dev/null +++ b/playbooks/UrlScan_IO_Dynamic_Analysis.py @@ -0,0 +1,619 @@ +""" +Accepts a URL for detonation analysis on the objects. Generates a global report and a per observable sub-report and normalized score. The score can be customized based on a variety of factors.\n\nRef: https://d3fend.mitre.org/technique/d3f:IdentifierReputationAnalysis/ +""" + + +import phantom.rules as phantom +import json +from datetime import datetime, timedelta + + +@phantom.playbook_block() +def on_start(container): + phantom.debug('on_start() called') + + # call 'url_input_filter' block + url_input_filter(container=container) + + return + +@phantom.playbook_block() +def url_input_filter(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("url_input_filter() called") + + ################################################################################ + # Determine branches based on provided inputs. + ################################################################################ + + # collect filtered artifact ids and results for 'if' condition 1 + matched_artifacts_1, matched_results_1 = phantom.condition( + container=container, + conditions=[ + ["playbook_input:url", "!=", ""] + ], + name="url_input_filter:condition_1") + + # call connected blocks if filtered artifacts or results + if matched_artifacts_1 or matched_results_1: + url_reputation(action=action, success=success, container=container, results=results, handle=handle, filtered_artifacts=matched_artifacts_1, filtered_results=matched_results_1) + + return + + +@phantom.playbook_block() +def url_reputation(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("url_reputation() called") + + # phantom.debug('Action: {0} {1}'.format(action['name'], ('SUCCEEDED' if success else 'FAILED'))) + + ################################################################################ + # Queries urlscan.io for information about the provided URL(s) + ################################################################################ + + playbook_input_url = phantom.collect2(container=container, datapath=["playbook_input:url"]) + + parameters = [] + + # build parameters list for 'url_reputation' call + for playbook_input_url_item in playbook_input_url: + if playbook_input_url_item[0] is not None: + parameters.append({ + "url": playbook_input_url_item[0], + "private": True, + "get_result": True, + }) + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.act("detonate url", parameters=parameters, name="url_reputation", assets=["urlscan.io"], callback=urlscanio_summary_filter) + + return + + +@phantom.playbook_block() +def urlscanio_summary_filter(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("urlscanio_summary_filter() called") + + ################################################################################ + # Filters successful url reputation results. + ################################################################################ + + # collect filtered artifact ids and results for 'if' condition 1 + matched_artifacts_1, matched_results_1 = phantom.condition( + container=container, + conditions=[ + ["url_reputation:action_result.status", "==", "success"] + ], + name="urlscanio_summary_filter:condition_1") + + # call connected blocks if filtered artifacts or results + if matched_artifacts_1 or matched_results_1: + urlscanio_error_code_filter(action=action, success=success, container=container, results=results, handle=handle, filtered_artifacts=matched_artifacts_1, filtered_results=matched_results_1) + + return + + +@phantom.playbook_block() +def normalize_score_url_with_error_code(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("normalize_score_url_with_error_code() called") + + ################################################################################ + # This block uses custom code for normalizing score. Adjust the logic as desired + # in the documented sections. + ################################################################################ + + url_reputation_result_data = phantom.collect2(container=container, datapath=["url_reputation:action_result.parameter.url","url_reputation:action_result.data.*.message","url_reputation:action_result.data.*.status","url_reputation:action_result.data.*.description"], action_results=results) + + url_reputation_parameter_url = [item[0] for item in url_reputation_result_data] + url_reputation_result_item_1 = [item[1] for item in url_reputation_result_data] + url_reputation_result_item_2 = [item[2] for item in url_reputation_result_data] + url_reputation_result_item_3 = [item[3] for item in url_reputation_result_data] + + normalize_score_url_with_error_code__url_score_object = None + normalize_score_url_with_error_code__scores = None + normalize_score_url_with_error_code__categories = None + normalize_score_url_with_error_code__confidence = None + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + url_reputation_data_message = [str(i or '') for i in url_reputation_result_item_1] + url_reputation_data_status = [str(i or '') for i in url_reputation_result_item_2] + url_reputation_data_description = [str(i or '') for i in url_reputation_result_item_3] + + + normalize_score_url_with_error_code__url_score_object = [] + normalize_score_url_with_error_code__scores = [] + normalize_score_url_with_error_code__categories = [] + + + #phantom.debug("url_reputation_parameter_url: {}".format(url_reputation_parameter_url)) + #phantom.debug("url_reputation_data_message: {}".format(url_reputation_data_message)) + #phantom.debug("url_reputation_data_status: {}".format(url_reputation_data_status)) + #phantom.debug("url_reputation_data_description: {}".format(url_reputation_data_description)) + + + + + urlscan_score_table = { + "0":"Legitimate", + "1":"Very_Safe", + "2":"Safe", + "3":"Probably_Safe", + "4":"Leans_Safe", + "5":"May_not_be_Safe", + "6":"Exercise_Caution", + "7":"Suspicious_or_Risky", + "8":"Possibly_Malicious", + "9":"Probably_Malicious", + "10":"Malicious", + "error_code_query" : "error code return, check the error code descriptions" + + } + + url_scan_io_error_code ={ + "blacklist" : "Blacklisted URL or Domain", + "spam" : "Spammy URL or Domain", + "invalid_hostname" : "Invalid Hostname URL or Domain", + "missing_url" : "Missing URL OR Domain Property", + "auth" : "HTTP basic auth information", + "not_be_resolved" : "Non-resolvable hostname (A, AAAA, CNAME)" + } + ## URLSCAN.io return error code especially if the url or domain was already in their blacklist database. + ## below are the common error code message base on their + ## - "Blacklisted domains and URLs" : requested to be blacklisted by their respective owners. + ## - "Spammy submissions" : of URLs known to be used only for spamming this service. + ## - "Invalid hostnames" : or invalid protocol schemes (FTP etc). + ## - "Missing URL property" : ... yes, it does happen. + ## - "Contains HTTP basic auth information" : ... yes, that happens as well. + ## - "Non-resolvable hostnames (A, AAAA, CNAME)" : which we will not even try to scan. + + blank_result = "--" + category = "" + score = "" + error_message = "" + + ## check if there is error code return upon URL reputation query + + for url_descp in url_reputation_data_description: + + for key, value in url_scan_io_error_code.items(): + if key.replace("_"," ").lower() in url_descp.lower(): + error_message = url_scan_io_error_code[key] + + # Attach final object + normalize_score_url_with_error_code__categories.append(error_message) + normalize_score_url_with_error_code__url_score_object.append({'score': urlscan_score_table['error_code_query'], 'confidence':"", 'score_id': "", "malicious_tag_verdicts": "", 'categories': error_message, "description": url_descp}) + normalize_score_url_with_error_code__scores.append(urlscan_score_table['error_code_query']) + + phantom.debug("normalize_score_url_with_error_code__url_score_object: {}".format(normalize_score_url_with_error_code__url_score_object)) + phantom.debug("normalize_score_url_with_error_code__scores: {}".format(normalize_score_url_with_error_code__scores)) + phantom.debug("normalize_score_url_with_error_code__categories: {}".format(normalize_score_url_with_error_code__categories)) + + + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.save_run_data(key="normalize_score_url_with_error_code:url_score_object", value=json.dumps(normalize_score_url_with_error_code__url_score_object)) + phantom.save_run_data(key="normalize_score_url_with_error_code:scores", value=json.dumps(normalize_score_url_with_error_code__scores)) + phantom.save_run_data(key="normalize_score_url_with_error_code:categories", value=json.dumps(normalize_score_url_with_error_code__categories)) + phantom.save_run_data(key="normalize_score_url_with_error_code:confidence", value=json.dumps(normalize_score_url_with_error_code__confidence)) + + error_code_format_report_url(container=container) + + return + + +@phantom.playbook_block() +def error_code_format_report_url(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("error_code_format_report_url() called") + + ################################################################################ + # Format a summary table with the information gathered from the playbook. + ################################################################################ + + template = """SOAR analyzed URL(s) using urlscan.io. The table below shows a summary of the information gathered.\n\n| URL | Normalized Score | Confidence |Categories | Report Link | Source |\n| --- | --- | --- | --- | --- |\n%%\n| `{0}` | {1} | {2} | {3} | {4} | urlscan.io |\n\n""" + + # parameter list for template variable replacement + parameters = [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_error_code:custom_function:confidence", + "normalize_score_url_with_error_code:custom_function:scores", + "normalize_score_url_with_error_code:custom_function:categories", + "url_reputation:action_result.data.*.task.reportURL" + ] + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + #phantom.debug(phantom.format(container=container, template=template, parameters=parameters, name="error_code_format_report_url")) + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.format(container=container, template=template, parameters=parameters, name="error_code_format_report_url") + + build_url_output_with_error_code(container=container) + + return + + +@phantom.playbook_block() +def urlscanio_error_code_filter(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("urlscanio_error_code_filter() called") + + ################################################################################ + # This filter is designed to avoid nonetype value in some list and dictionary + # object due to error code return value of urlscan.io especially if the URL or + # domain is in their blacklist database + ################################################################################ + + # collect filtered artifact ids and results for 'if' condition 1 + matched_artifacts_1, matched_results_1 = phantom.condition( + container=container, + conditions=[ + ["filtered-data:urlscanio_summary_filter:condition_1:url_reputation:action_result.data.*.status", "==", 400] + ], + name="urlscanio_error_code_filter:condition_1") + + # call connected blocks if filtered artifacts or results + if matched_artifacts_1 or matched_results_1: + normalize_score_url_with_error_code(action=action, success=success, container=container, results=results, handle=handle, filtered_artifacts=matched_artifacts_1, filtered_results=matched_results_1) + + # collect filtered artifact ids and results for 'if' condition 2 + matched_artifacts_2, matched_results_2 = phantom.condition( + container=container, + conditions=[ + ["filtered-data:urlscanio_summary_filter:condition_1:url_reputation:action_result.data.*.status", "!=", 400] + ], + name="urlscanio_error_code_filter:condition_2") + + # call connected blocks if filtered artifacts or results + if matched_artifacts_2 or matched_results_2: + normalize_score_url_with_no_error_code(action=action, success=success, container=container, results=results, handle=handle, filtered_artifacts=matched_artifacts_2, filtered_results=matched_results_2) + + return + + +@phantom.playbook_block() +def normalize_score_url_with_no_error_code(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("normalize_score_url_with_no_error_code() called") + + ################################################################################ + # This block uses custom code for normalizing score. Adjust the logic as desired + # in the documented sections. + ################################################################################ + + url_reputation_result_data = phantom.collect2(container=container, datapath=["url_reputation:action_result.data.*.verdicts.overall","url_reputation:action_result.data.*.verdicts.urlscan","url_reputation:action_result.data.*.verdicts.engines","url_reputation:action_result.data.*.verdicts.community"], action_results=results) + + url_reputation_result_item_0 = [item[0] for item in url_reputation_result_data] + url_reputation_result_item_1 = [item[1] for item in url_reputation_result_data] + url_reputation_result_item_2 = [item[2] for item in url_reputation_result_data] + url_reputation_result_item_3 = [item[3] for item in url_reputation_result_data] + + normalize_score_url_with_no_error_code__url_score_object = None + normalize_score_url_with_no_error_code__scores = None + normalize_score_url_with_no_error_code__categories = None + normalize_score_url_with_no_error_code__confidence = None + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + + normalize_score_url_with_no_error_code__url_score_object = [] + normalize_score_url_with_no_error_code__scores = [] + normalize_score_url_with_no_error_code__categories = [] + + url_reputation_verdicts_overall_dict = [(i or {}) for i in url_reputation_result_item_0] + url_reputation_verdicts_urlscan_dict = [(i or {}) for i in url_reputation_result_item_1] + url_reputation_verdicts_engine_dict = [(i or {}) for i in url_reputation_result_item_2] + url_reputation_verdicts_community_dict = [(i or {}) for i in url_reputation_result_item_3] + + #phantom.debug("url_reputation_verdicts_overall_dict: {}".format(url_reputation_verdicts_overall_dict)) + #phantom.debug("url_reputation_verdicts_urlscan_dict: {}".format(url_reputation_verdicts_urlscan_dict)) + #phantom.debug("url_reputation_verdicts_engine_dict: {}".format(url_reputation_verdicts_engine_dict)) + #phantom.debug("url_reputation_verdicts_community_dict: {}".format(url_reputation_verdicts_community_dict)) + + + urlscan_score_table = { + "0":"Legitimate", + "1":"Very_Safe", + "2":"Safe", + "3":"Probably_Safe", + "4":"Leans_Safe", + "5":"May_not_be_Safe", + "6":"Exercise_Caution", + "7":"Suspicious_or_Risky", + "8":"Possibly_Malicious", + "9":"Probably_Malicious", + "10":"Malicious" + } + + ## if there is no error code, urlscan.io will continue to detonate the URL and query scores in several verdicts object + + ## Normalize reputation on a -100 (legitimate) to 100 point scale based on number of malicious and suspicious divided by different urlscan.io verdict objects. + ## This can be adjusted to include whatever logic is desired. + + for i in range(0,len(url_reputation_verdicts_overall_dict)): + if url_reputation_verdicts_overall_dict[i] != {} or url_reputation_verdicts_urlscan_dict[i] != {} or url_reputation_verdicts_engine_dict[i] != {} or url_reputation_verdicts_community_dict[i] != {}: + summary_score = url_reputation_verdicts_overall_dict[i]['score'] + url_reputation_verdicts_urlscan_dict[i]['score'] + url_reputation_verdicts_engine_dict[i]['score'] + url_reputation_verdicts_community_dict[i]['score'] + summary_malicious_verdicts = url_reputation_verdicts_overall_dict[i]['malicious'] or url_reputation_verdicts_urlscan_dict[i]['malicious'] or url_reputation_verdicts_engine_dict[i]['malicious'] or url_reputation_verdicts_community_dict[i]['malicious'] + summary_of_malicious_tag = int(url_reputation_verdicts_overall_dict[i]['malicious']) + int(url_reputation_verdicts_urlscan_dict[i]['malicious']) + int(url_reputation_verdicts_engine_dict[i]['malicious']) + int(url_reputation_verdicts_community_dict[i]['malicious']) + summary_categories = url_reputation_verdicts_overall_dict[i]['categories'] + url_reputation_verdicts_urlscan_dict[i]['categories'] + url_reputation_verdicts_engine_dict[i]['categories'] + url_reputation_verdicts_community_dict[i]['categories'] + + + ## customized score id calculation + + log_result = (summary_score/4) # avg score from different urlscan.io score object (engine_score, overall_verdict_score, urlscan_verdicts_score and community score) + score_id = int(log_result) + + + if score_id < -50: + score_id = "0" + elif score_id < 0 and score_id >= -50: + score_id = "1" + elif score_id >= 0 and score_id <= 10: + score_id = "2" + elif score_id > 10 and score_id <= 20: + score_id = "3" + elif score_id > 20 and score_id <= 30: + score_id = "4" + elif score_id > 30 and score_id <= 40: + score_id = "5" + elif score_id > 40 and score_id <= 50: + score_id = "6" + elif score_id > 50 and score_id <= 60: + score_id = "7" + elif score_id > 70 and score_id <= 80: + score_id = "8" + elif score_id > 80 and score_id <= 90: + score_id = "9" + elif score_id > 90 and score_id <= 100: + score_id = "10" + + score = urlscan_score_table[str(score_id)] + + malicious_tag_stats = (summary_of_malicious_tag, 4) + + # Attach final object + normalize_score_url_with_no_error_code__categories.append(summary_categories) + normalize_score_url_with_no_error_code__url_score_object.append({'score': score, 'confidence':log_result, 'score_id': score_id, "malicious_tag_verdicts": summary_malicious_verdicts, "malicious_tag_stats": malicious_tag_stats , 'categories': summary_categories, "description": ""}) + normalize_score_url_with_no_error_code__scores.append(score) + + phantom.debug("normalize_score_url_with_no_error_code__categories: {}".format(normalize_score_url_with_no_error_code__categories)) + phantom.debug("normalize_score_url_with_no_error_code__url_score_object: {}".format(normalize_score_url_with_no_error_code__url_score_object)) + phantom.debug("normalize_score_url_with_no_error_code__scores: {}".format(normalize_score_url_with_no_error_code__scores)) + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.save_run_data(key="normalize_score_url_with_no_error_code:url_score_object", value=json.dumps(normalize_score_url_with_no_error_code__url_score_object)) + phantom.save_run_data(key="normalize_score_url_with_no_error_code:scores", value=json.dumps(normalize_score_url_with_no_error_code__scores)) + phantom.save_run_data(key="normalize_score_url_with_no_error_code:categories", value=json.dumps(normalize_score_url_with_no_error_code__categories)) + phantom.save_run_data(key="normalize_score_url_with_no_error_code:confidence", value=json.dumps(normalize_score_url_with_no_error_code__confidence)) + + no_error_code_format_report_url(container=container) + + return + + +@phantom.playbook_block() +def no_error_code_format_report_url(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("no_error_code_format_report_url() called") + + ################################################################################ + # Format a summary table with the information gathered from the playbook. + ################################################################################ + + template = """SOAR analyzed URL(s) using urlscan.io. The table below shows a summary of the information gathered.\n\n| URL | Normalized Score |Confidence | Categories | Report Link | Source |\n| --- | --- | --- | --- | --- |\n%%\n| `{0}` | {1} | {2} | {3} |{4} | urlscan.io |\n""" + + # parameter list for template variable replacement + parameters = [ + "url_reputation:action_result.parameter.url", + "normalize_score_url_with_no_error_code:custom_function:scores", + "normalize_score_url_with_no_error_code:custom_function:confidence", + "normalize_score_url_with_no_error_code:custom_function:categories", + "url_reputation:action_result.data.*.task.reportURL" + ] + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + #phantom.debug(phantom.format(container=container, template=template, parameters=parameters, name="no_error_code_format_report_url")) + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.format(container=container, template=template, parameters=parameters, name="no_error_code_format_report_url") + + build_url_output_with_no_error_code(container=container) + + return + + +@phantom.playbook_block() +def build_url_output_with_error_code(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("build_url_output_with_error_code() called") + + ################################################################################ + # This block uses custom code to generate an observable dictionary to output into + # the observables data path. + ################################################################################ + + url_reputation_result_data = phantom.collect2(container=container, datapath=["url_reputation:action_result.parameter.url","url_reputation:action_result.data.*.task.reportURL"], action_results=results) + normalize_score_url_with_error_code__url_score_object = json.loads(_ if (_ := phantom.get_run_data(key="normalize_score_url_with_error_code:url_score_object")) != "" else "null") # pylint: disable=used-before-assignment + + url_reputation_parameter_url = [item[0] for item in url_reputation_result_data] + url_reputation_result_item_1 = [item[1] for item in url_reputation_result_data] + + build_url_output_with_error_code__observable_array = None + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + # Write your custom code here... + from urllib.parse import urlparse + build_url_output_with_error_code__observable_array = [] + + # Build URL + url_scan_io_task_reporturl = [str(i or 'no report url') for i in url_reputation_result_item_1] + url_scan_io_parameter_url = [str(i or '') for i in url_reputation_parameter_url] + url_scan_io_url_score_object = normalize_score_url_with_error_code__url_score_object + + phantom.debug("url_reputation_parameter_url: {}".format(url_reputation_parameter_url)) + #phantom.debug("url_reputation_result_item_1: {}".format(url_reputation_result_item_1)) + #phantom.debug("normalize_score_url_with_error_code__url_score_object: {}".format(normalize_score_url_with_error_code__url_score_object)) + for url, external_id, url_object in zip(url_scan_io_parameter_url, url_scan_io_task_reporturl, url_scan_io_url_score_object): + parsed_url = urlparse(url) + phantom.debug("{} {} {} parsed_url: {}".format(url, external_id, url_object, parsed_url)) + observable_object = { + "value": url, + "type": "url", + "sandbox": { + "score_id": url_object['score_id'], + "score": url_object['score'], + "confidence": url_object['confidence'] + }, + "attributes": { + "hostname": parsed_url.hostname, + "scheme": parsed_url.scheme + }, + "categories": url_object['categories'], + "description" :url_object['description'], + "source": "urlscan.io", + "source_link": f"{external_id}" + } + + if parsed_url.path: + observable_object['attributes']['path'] = parsed_url.path + if parsed_url.query: + observable_object['attributes']['query'] = parsed_url.query + if parsed_url.port: + observable_object['attributes']['port'] = parsed_url.port + + + build_url_output_with_error_code__observable_array.append(observable_object) + #phantom.debug("build_url_output_with_error_code__observable_array: {}".format(build_url_output_with_error_code__observable_array)) + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.save_run_data(key="build_url_output_with_error_code:observable_array", value=json.dumps(build_url_output_with_error_code__observable_array)) + + return + + +@phantom.playbook_block() +def build_url_output_with_no_error_code(action=None, success=None, container=None, results=None, handle=None, filtered_artifacts=None, filtered_results=None, custom_function=None, **kwargs): + phantom.debug("build_url_output_with_no_error_code() called") + + ################################################################################ + # This block uses custom code to generate an observable dictionary to output into + # the observables data path. + ################################################################################ + + url_reputation_result_data = phantom.collect2(container=container, datapath=["url_reputation:action_result.parameter.url","url_reputation:action_result.data.*.task.reportURL"], action_results=results) + normalize_score_url_with_no_error_code__url_score_object = json.loads(_ if (_ := phantom.get_run_data(key="normalize_score_url_with_no_error_code:url_score_object")) != "" else "null") # pylint: disable=used-before-assignment + + url_reputation_parameter_url = [item[0] for item in url_reputation_result_data] + url_reputation_result_item_1 = [item[1] for item in url_reputation_result_data] + + build_url_output_with_no_error_code__observable_array = None + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + from urllib.parse import urlparse + build_url_output_with_no_error_code__observable_array = [] + + # Build URL + phantom.debug(url_reputation_parameter_url) + for url, external_id, url_object in zip(url_reputation_parameter_url, url_reputation_result_item_1, normalize_score_url_with_no_error_code__url_score_object): + parsed_url = urlparse(url) + phantom.debug("{} {} {} parsed_url: {}".format(url, external_id, url_object, parsed_url)) + observable_object = { + "value": url, + "type": "url", + "sandbox": { + "score_id": url_object['score_id'], + "score": url_object['score'], + "confidence": url_object['confidence'] + }, + "attributes": { + "hostname": parsed_url.hostname, + "scheme": parsed_url.scheme + }, + "categories": url_object['categories'], + "description" :url_object['description'], + "source": "urlscan.io", + "source_link": f"{external_id}" + } + if parsed_url.path: + observable_object['attributes']['path'] = parsed_url.path + if parsed_url.query: + observable_object['attributes']['query'] = parsed_url.query + if parsed_url.port: + observable_object['attributes']['port'] = parsed_url.port + + build_url_output_with_no_error_code__observable_array.append(observable_object) + #phantom.debug("build_url_output_with_no_error_code__observable_array: {}".format(build_url_output_with_no_error_code__observable_array)) + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.save_run_data(key="build_url_output_with_no_error_code:observable_array", value=json.dumps(build_url_output_with_no_error_code__observable_array)) + + return + + +@phantom.playbook_block() +def on_finish(container, summary): + phantom.debug("on_finish() called") + + build_url_output_with_error_code__observable_array = json.loads(_ if (_ := phantom.get_run_data(key="build_url_output_with_error_code:observable_array")) != "" else "null") # pylint: disable=used-before-assignment + build_url_output_with_no_error_code__observable_array = json.loads(_ if (_ := phantom.get_run_data(key="build_url_output_with_no_error_code:observable_array")) != "" else "null") # pylint: disable=used-before-assignment + + observable_combined_value = phantom.concatenate(build_url_output_with_error_code__observable_array, build_url_output_with_no_error_code__observable_array) + + output = { + "observable": observable_combined_value, + } + + ################################################################################ + ## Custom Code Start + ################################################################################ + + # Write your custom code here... + no_error_code_format_report_url = phantom.get_format_data(name="no_error_code_format_report_url") + error_code_format_report_url = phantom.get_format_data(name="error_code_format_report_url") + markdown_report_combined_value = phantom.concatenate(no_error_code_format_report_url, error_code_format_report_url) + output['markdown_report'] = markdown_report_combined_value + ################################################################################ + ## Custom Code End + ################################################################################ + + phantom.save_playbook_output_data(output=output) + + return + diff --git a/playbooks/UrlScan_IO_Dynamic_Analysis.yml b/playbooks/UrlScan_IO_Dynamic_Analysis.yml new file mode 100644 index 0000000000..e68eab1e7e --- /dev/null +++ b/playbooks/UrlScan_IO_Dynamic_Analysis.yml @@ -0,0 +1,25 @@ +name: UrlScan IO Dynamic Analysis +id: a1173c28-7b33-4a56-9d7f-5dbbca595cb0 +version: 1 +date: '2023-03-23' +author: Teoderick Contreras, Splunk +type: Investigation +description: "Accepts a url link, IP, or domain to be detonated using urlscan.io API connector." +playbook: UrlScan_IO_Dynamic_Analysis +how_to_implement: This input playbook requires the urlscan.io API connector to be configured. + It is designed to work in conjunction with the Dynamic Attribute Lookup playbook or other playbooks in the same style. +references: [] +app_list: + - urlscan.io +tags: + platform_tags: + - reputation + - url + - domain + - sandbox + - ip + playbook_type: Input + vpe_type: Modern + playbook_fields: [] + product: + - Splunk SOAR