mirror of
https://github.com/FutureComputing4AI/EMBER2024
synced 2026-06-08 11:06:05 +00:00
RJ
310 lines
34 KiB
Plaintext
310 lines
34 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"id": "320d6c51",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"RendererRegistry.enable('default')"
|
|
]
|
|
},
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import os\n",
|
|
"import thrember\n",
|
|
"import numpy as np\n",
|
|
"import pandas as pd\n",
|
|
"import polars as pl\n",
|
|
"import altair as alt\n",
|
|
"import lightgbm as lgb\n",
|
|
"import matplotlib.pylab as plt\n",
|
|
"from sklearn.metrics import roc_auc_score, roc_curve\n",
|
|
"alt.renderers.enable('default')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"id": "af906f0c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"data_dir = \"/data/EMBER2024/\" # change this to where you unzipped the download"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "66713c94",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"train_df, test_df, challenge_df = thrember.read_metadata(data_dir)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"id": "0b72ccbf",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"(3232000, 16)\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"\n",
|
|
"<style>\n",
|
|
" #altair-viz-547a67a78337426bb42f2a9c94ea2eb4.vega-embed {\n",
|
|
" width: 100%;\n",
|
|
" display: flex;\n",
|
|
" }\n",
|
|
"\n",
|
|
" #altair-viz-547a67a78337426bb42f2a9c94ea2eb4.vega-embed details,\n",
|
|
" #altair-viz-547a67a78337426bb42f2a9c94ea2eb4.vega-embed details summary {\n",
|
|
" position: relative;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<div id=\"altair-viz-547a67a78337426bb42f2a9c94ea2eb4\"></div>\n",
|
|
"<script type=\"text/javascript\">\n",
|
|
" var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
|
|
" (function(spec, embedOpt){\n",
|
|
" let outputDiv = document.currentScript.previousElementSibling;\n",
|
|
" if (outputDiv.id !== \"altair-viz-547a67a78337426bb42f2a9c94ea2eb4\") {\n",
|
|
" outputDiv = document.getElementById(\"altair-viz-547a67a78337426bb42f2a9c94ea2eb4\");\n",
|
|
" }\n",
|
|
"\n",
|
|
" const paths = {\n",
|
|
" \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
|
|
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
|
|
" \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.20.1?noext\",\n",
|
|
" \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
|
|
" };\n",
|
|
"\n",
|
|
" function maybeLoadScript(lib, version) {\n",
|
|
" var key = `${lib.replace(\"-\", \"\")}_version`;\n",
|
|
" return (VEGA_DEBUG[key] == version) ?\n",
|
|
" Promise.resolve(paths[lib]) :\n",
|
|
" new Promise(function(resolve, reject) {\n",
|
|
" var s = document.createElement('script');\n",
|
|
" document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
|
|
" s.async = true;\n",
|
|
" s.onload = () => {\n",
|
|
" VEGA_DEBUG[key] = version;\n",
|
|
" return resolve(paths[lib]);\n",
|
|
" };\n",
|
|
" s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
|
|
" s.src = paths[lib];\n",
|
|
" });\n",
|
|
" }\n",
|
|
"\n",
|
|
" function showError(err) {\n",
|
|
" outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
|
|
" throw err;\n",
|
|
" }\n",
|
|
"\n",
|
|
" function displayChart(vegaEmbed) {\n",
|
|
" vegaEmbed(outputDiv, spec, embedOpt)\n",
|
|
" .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
|
|
" }\n",
|
|
"\n",
|
|
" if(typeof define === \"function\" && define.amd) {\n",
|
|
" requirejs.config({paths});\n",
|
|
" let deps = [\"vega-embed\"];\n",
|
|
" require(deps, displayChart, err => showError(`Error loading script: ${err.message}`));\n",
|
|
" } else {\n",
|
|
" maybeLoadScript(\"vega\", \"5\")\n",
|
|
" .then(() => maybeLoadScript(\"vega-lite\", \"5.20.1\"))\n",
|
|
" .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
|
|
" .catch(showError)\n",
|
|
" .then(() => displayChart(vegaEmbed));\n",
|
|
" }\n",
|
|
" })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300}}, \"data\": {\"name\": \"data-1d9f965aa68bb2e806420b939d5894a1\"}, \"mark\": {\"type\": \"bar\"}, \"encoding\": {\"color\": {\"field\": \"file_type\", \"legend\": {\"values\": [\"Win32\", \"Win64\", \"Dot_Net\", \"APK\", \"ELF\", \"PDF\"]}, \"scale\": {\"range\": [\"#4c78a8\", \"#54a24b\", \"#f58518\", \"#88d27a\", \"#9ecae9\", \"#ffbf79\"]}, \"type\": \"nominal\"}, \"x\": {\"axis\": {\"title\": \"Week First Seen\"}, \"field\": \"week\", \"type\": \"ordinal\"}, \"y\": {\"axis\": {\"title\": \"File Type\"}, \"field\": \"count\", \"type\": \"quantitative\"}}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.20.1.json\", \"datasets\": {\"data-1d9f965aa68bb2e806420b939d5894a1\": [{\"file_type\": \"Win32\", \"week\": 37, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 0, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 21, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 23, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 26, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 11, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 38, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 23, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 51, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 26, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 0, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 14, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 16, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 20, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 17, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 44, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 61, \"count\": 29999}, {\"file_type\": \"Win64\", \"week\": 39, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 4, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 32, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 51, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 5, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 61, \"count\": 602}, {\"file_type\": \"Dot_Net\", \"week\": 49, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 21, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 24, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 57, \"count\": 30010}, {\"file_type\": \"Win64\", \"week\": 30, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 29, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 24, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 21, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 1, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 35, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 3, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 40, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 3, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 35, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 21, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 52, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 57, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 50, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 57, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 7, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 44, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 47, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 24, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 22, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 54, \"count\": 5090}, {\"file_type\": \"Dot_Net\", \"week\": 9, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 22, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 20, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 56, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 10, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 31, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 45, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 49, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 27, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 62, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 51, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 48, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 31, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 10, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 54, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 6, \"count\": 10000}, {\"file_type\": \"APK\", \"week\": 12, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 23, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 53, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 25, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 28, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 48, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 6, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 56, \"count\": 4994}, {\"file_type\": \"APK\", \"week\": 37, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 1, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 36, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 0, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 37, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 12, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 59, \"count\": 5068}, {\"file_type\": \"ELF\", \"week\": 39, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 60, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 43, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 15, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 17, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 36, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 11, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 46, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 44, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 23, \"count\": 10000}, {\"file_type\": \"APK\", \"week\": 2, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 58, \"count\": 511}, {\"file_type\": \"APK\", \"week\": 46, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 25, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 25, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 35, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 23, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 13, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 4, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 55, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 34, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 17, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 41, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 28, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 29, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 39, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 8, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 41, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 33, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 33, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 7, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 16, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 40, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 55, \"count\": 4961}, {\"file_type\": \"Win64\", \"week\": 19, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 63, \"count\": 491}, {\"file_type\": \"PDF\", \"week\": 37, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 47, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 51, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 20, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 63, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 28, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 1, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 14, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 18, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 39, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 40, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 46, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 35, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 43, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 14, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 39, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 54, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 51, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 1, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 48, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 30, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 31, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 6, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 10, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 26, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 58, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 34, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 18, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 58, \"count\": 29988}, {\"file_type\": \"Win64\", \"week\": 10, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 0, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 11, \"count\": 10000}, {\"file_type\": \"APK\", \"week\": 34, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 60, \"count\": 4979}, {\"file_type\": \"APK\", \"week\": 52, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 44, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 36, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 53, \"count\": 511}, {\"file_type\": \"Win64\", \"week\": 59, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 6, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 41, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 57, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 25, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 58, \"count\": 4922}, {\"file_type\": \"ELF\", \"week\": 42, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 8, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 23, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 16, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 50, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 2, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 52, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 33, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 53, \"count\": 10003}, {\"file_type\": \"Dot_Net\", \"week\": 63, \"count\": 4956}, {\"file_type\": \"ELF\", \"week\": 24, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 55, \"count\": 29999}, {\"file_type\": \"APK\", \"week\": 22, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 11, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 19, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 57, \"count\": 5031}, {\"file_type\": \"PDF\", \"week\": 11, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 35, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 8, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 44, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 29, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 47, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 30, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 38, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 47, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 26, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 32, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 25, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 46, \"count\": 10000}, {\"file_type\": \"APK\", \"week\": 27, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 14, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 8, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 11, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 60, \"count\": 481}, {\"file_type\": \"Win32\", \"week\": 18, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 31, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 5, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 13, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 56, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 42, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 16, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 34, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 10, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 28, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 42, \"count\": 10000}, {\"file_type\": \"Win64\", \"week\": 14, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 29, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 0, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 59, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 18, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 3, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 19, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 2, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 45, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 27, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 33, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 30, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 13, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 43, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 56, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 44, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 6, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 46, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 24, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 63, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 25, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 3, \"count\": 4000}, {\"file_type\": \"Dot_Net\", \"week\": 5, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 59, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 32, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 49, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 12, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 5, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 38, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 16, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 48, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 7, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 15, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 27, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 3, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 62, \"count\": 4938}, {\"file_type\": \"ELF\", \"week\": 49, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 22, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 15, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 3, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 9, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 21, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 51, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 13, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 8, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 20, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 30, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 9, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 63, \"count\": 29999}, {\"file_type\": \"Win64\", \"week\": 50, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 45, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 54, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 32, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 53, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 60, \"count\": 30002}, {\"file_type\": \"Dot_Net\", \"week\": 40, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 40, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 53, \"count\": 5045}, {\"file_type\": \"PDF\", \"week\": 29, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 38, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 41, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 4, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 52, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 59, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 13, \"count\": 10000}, {\"file_type\": \"APK\", \"week\": 47, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 2, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 52, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 62, \"count\": 10003}, {\"file_type\": \"Win64\", \"week\": 48, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 26, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 15, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 42, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 56, \"count\": 485}, {\"file_type\": \"APK\", \"week\": 4, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 33, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 19, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 28, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 59, \"count\": 499}, {\"file_type\": \"Win32\", \"week\": 5, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 37, \"count\": 5000}, {\"file_type\": \"PDF\", \"week\": 9, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 17, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 12, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 27, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 35, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 61, \"count\": 9999}, {\"file_type\": \"PDF\", \"week\": 39, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 34, \"count\": 30000}, {\"file_type\": \"Dot_Net\", \"week\": 6, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 1, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 26, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 62, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 2, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 62, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 45, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 12, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 17, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 7, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 58, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 38, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 8, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 55, \"count\": 9998}, {\"file_type\": \"APK\", \"week\": 61, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 27, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 47, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 32, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 12, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 4, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 9, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 22, \"count\": 30000}, {\"file_type\": \"Win64\", \"week\": 58, \"count\": 9999}, {\"file_type\": \"PDF\", \"week\": 49, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 17, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 4, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 7, \"count\": 10000}, {\"file_type\": \"Win32\", \"week\": 52, \"count\": 30000}, {\"file_type\": \"Win32\", \"week\": 9, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 14, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 46, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 13, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 49, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 18, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 16, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 54, \"count\": 533}, {\"file_type\": \"PDF\", \"week\": 45, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 5, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 36, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 7, \"count\": 5000}, {\"file_type\": \"Win32\", \"week\": 54, \"count\": 30003}, {\"file_type\": \"Win64\", \"week\": 63, \"count\": 9997}, {\"file_type\": \"ELF\", \"week\": 48, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 41, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 30, \"count\": 1000}, {\"file_type\": \"ELF\", \"week\": 57, \"count\": 487}, {\"file_type\": \"ELF\", \"week\": 38, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 10, \"count\": 500}, {\"file_type\": \"Win64\", \"week\": 18, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 19, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 61, \"count\": 5016}, {\"file_type\": \"Win64\", \"week\": 21, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 32, \"count\": 5000}, {\"file_type\": \"APK\", \"week\": 0, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 37, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 53, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 1, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 43, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 43, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 60, \"count\": 4000}, {\"file_type\": \"Win32\", \"week\": 24, \"count\": 30000}, {\"file_type\": \"ELF\", \"week\": 55, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 31, \"count\": 1000}, {\"file_type\": \"PDF\", \"week\": 61, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 36, \"count\": 10000}, {\"file_type\": \"Dot_Net\", \"week\": 19, \"count\": 5000}, {\"file_type\": \"Dot_Net\", \"week\": 43, \"count\": 5000}, {\"file_type\": \"ELF\", \"week\": 34, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 50, \"count\": 500}, {\"file_type\": \"APK\", \"week\": 42, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 55, \"count\": 1000}, {\"file_type\": \"Dot_Net\", \"week\": 20, \"count\": 5000}, {\"file_type\": \"Win64\", \"week\": 15, \"count\": 10000}, {\"file_type\": \"PDF\", \"week\": 22, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 29, \"count\": 4000}, {\"file_type\": \"PDF\", \"week\": 50, \"count\": 1000}, {\"file_type\": \"Win32\", \"week\": 50, \"count\": 30000}, {\"file_type\": \"APK\", \"week\": 15, \"count\": 4000}, {\"file_type\": \"APK\", \"week\": 41, \"count\": 4000}, {\"file_type\": \"Win64\", \"week\": 33, \"count\": 10000}, {\"file_type\": \"ELF\", \"week\": 40, \"count\": 500}, {\"file_type\": \"ELF\", \"week\": 45, \"count\": 500}, {\"file_type\": \"PDF\", \"week\": 56, \"count\": 1000}, {\"file_type\": \"Win64\", \"week\": 60, \"count\": 10001}, {\"file_type\": \"ELF\", \"week\": 62, \"count\": 400}, {\"file_type\": \"ELF\", \"week\": 20, \"count\": 500}, {\"file_type\": \"Win32\", \"week\": 42, \"count\": 30000}, {\"file_type\": \"PDF\", \"week\": 28, \"count\": 1000}, {\"file_type\": \"APK\", \"week\": 36, \"count\": 4000}, {\"file_type\": \"ELF\", \"week\": 2, \"count\": 500}, {\"file_type\": \"Dot_Net\", \"week\": 31, \"count\": 5000}]}}, {\"mode\": \"vega-lite\"});\n",
|
|
"</script>"
|
|
],
|
|
"text/plain": [
|
|
"alt.Chart(...)"
|
|
]
|
|
},
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Add a 'week' column to the dataframe\n",
|
|
"plotdf = pl.concat([train_df, test_df])\n",
|
|
"start_date = pd.Timestamp(\"2023-09-24\")\n",
|
|
"plotdf = plotdf.with_columns(\n",
|
|
" pl.from_epoch(\"first_submission_date\", time_unit=\"s\").alias(\"first_submission_dt\")\n",
|
|
")\n",
|
|
"plotdf = plotdf.with_columns(\n",
|
|
" (\n",
|
|
" (pl.col(\"first_submission_dt\") - pl.lit(start_date)).dt.total_days() // 7\n",
|
|
" ).cast(pl.Int64).alias(\"week\")\n",
|
|
")\n",
|
|
"\n",
|
|
"print(plotdf.shape)\n",
|
|
"\n",
|
|
"# Plot file types across weeks\n",
|
|
"gbdf = plotdf.group_by([\"file_type\", \"week\"]).agg(pl.len().alias(\"count\"))\n",
|
|
"alt.Chart(gbdf).mark_bar().encode(\n",
|
|
" alt.X('week:O', axis=alt.Axis(title='Week First Seen')),\n",
|
|
" alt.Y('count:Q', axis=alt.Axis(title='File Type')),\n",
|
|
" alt.Color('file_type:N', scale=alt.Scale(range=[\"#4c78a8\", \"#54a24b\", \"#f58518\", \"#88d27a\", \"#9ecae9\", \"#ffbf79\"]),\n",
|
|
" legend=alt.Legend(values=[\"Win32\", \"Win64\", \"Dot_Net\", \"APK\", \"ELF\", \"PDF\"]))\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "501106eb",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div><style>\n",
|
|
".dataframe > thead > tr,\n",
|
|
".dataframe > tbody > tr {\n",
|
|
" text-align: right;\n",
|
|
" white-space: pre-wrap;\n",
|
|
"}\n",
|
|
"</style>\n",
|
|
"<small>shape: (10, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>family</th><th>count</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>"berbew"</td><td>174481</td></tr><tr><td>"wacatac"</td><td>81478</td></tr><tr><td>"expiro"</td><td>74339</td></tr><tr><td>"cosmu"</td><td>53965</td></tr><tr><td>"xmrig"</td><td>28903</td></tr><tr><td>"upatre"</td><td>25296</td></tr><tr><td>"sfone"</td><td>22177</td></tr><tr><td>"glupteba"</td><td>21670</td></tr><tr><td>"grandoreiro"</td><td>20551</td></tr><tr><td>"flystudio"</td><td>18141</td></tr></tbody></table></div>"
|
|
],
|
|
"text/plain": [
|
|
"shape: (10, 2)\n",
|
|
"┌─────────────┬────────┐\n",
|
|
"│ family ┆ count │\n",
|
|
"│ --- ┆ --- │\n",
|
|
"│ str ┆ u32 │\n",
|
|
"╞═════════════╪════════╡\n",
|
|
"│ berbew ┆ 174481 │\n",
|
|
"│ wacatac ┆ 81478 │\n",
|
|
"│ expiro ┆ 74339 │\n",
|
|
"│ cosmu ┆ 53965 │\n",
|
|
"│ xmrig ┆ 28903 │\n",
|
|
"│ upatre ┆ 25296 │\n",
|
|
"│ sfone ┆ 22177 │\n",
|
|
"│ glupteba ┆ 21670 │\n",
|
|
"│ grandoreiro ┆ 20551 │\n",
|
|
"│ flystudio ┆ 18141 │\n",
|
|
"└─────────────┴────────┘"
|
|
]
|
|
},
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Get number of occurrences of each family\n",
|
|
"family_counts = plotdf.select(\n",
|
|
" pl.col(\"family\").value_counts().alias(\"family_counts\")\n",
|
|
").unnest(\"family_counts\")\n",
|
|
"family_counts = (\n",
|
|
" plotdf.filter(pl.col(\"family\").is_not_null())\n",
|
|
" .select(pl.col(\"family\").value_counts())\n",
|
|
" .unnest(\"family\")\n",
|
|
" .sort(\"count\", descending=True)\n",
|
|
")\n",
|
|
"family_counts.head(10)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"id": "d6ab49b7",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div><style>\n",
|
|
".dataframe > thead > tr,\n",
|
|
".dataframe > tbody > tr {\n",
|
|
" text-align: right;\n",
|
|
" white-space: pre-wrap;\n",
|
|
"}\n",
|
|
"</style>\n",
|
|
"<small>shape: (10, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>behavior</th><th>count</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>"backdoor"</td><td>228349</td></tr><tr><td>"virus"</td><td>121971</td></tr><tr><td>"worm"</td><td>76115</td></tr><tr><td>"downloader"</td><td>61523</td></tr><tr><td>"spyware"</td><td>55780</td></tr><tr><td>"coinminer"</td><td>37680</td></tr><tr><td>"dropper"</td><td>33291</td></tr><tr><td>"adware"</td><td>24867</td></tr><tr><td>"phishing"</td><td>21782</td></tr><tr><td>"ransom"</td><td>16279</td></tr></tbody></table></div>"
|
|
],
|
|
"text/plain": [
|
|
"shape: (10, 2)\n",
|
|
"┌────────────┬────────┐\n",
|
|
"│ behavior ┆ count │\n",
|
|
"│ --- ┆ --- │\n",
|
|
"│ str ┆ u32 │\n",
|
|
"╞════════════╪════════╡\n",
|
|
"│ backdoor ┆ 228349 │\n",
|
|
"│ virus ┆ 121971 │\n",
|
|
"│ worm ┆ 76115 │\n",
|
|
"│ downloader ┆ 61523 │\n",
|
|
"│ spyware ┆ 55780 │\n",
|
|
"│ coinminer ┆ 37680 │\n",
|
|
"│ dropper ┆ 33291 │\n",
|
|
"│ adware ┆ 24867 │\n",
|
|
"│ phishing ┆ 21782 │\n",
|
|
"│ ransom ┆ 16279 │\n",
|
|
"└────────────┴────────┘"
|
|
]
|
|
},
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"# Get number of occurrences of each behavior tag\n",
|
|
"plotdf_explode = plotdf.filter(pl.col(\"behavior\").list.len() > 0).explode(\"behavior\")\n",
|
|
"behavior_counts = (plotdf_explode.group_by(\"behavior\").agg(pl.len().alias(\"count\")).sort(\"count\", descending=True))\n",
|
|
"behavior_counts.head(10)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "935c3c7b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|