{ "cells": [ { "cell_type": "code", "execution_count": 8, "id": "320d6c51", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "RendererRegistry.enable('default')" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import os\n", "import thrember\n", "import numpy as np\n", "import pandas as pd\n", "import polars as pl\n", "import altair as alt\n", "import lightgbm as lgb\n", "import matplotlib.pylab as plt\n", "from sklearn.metrics import roc_auc_score, roc_curve\n", "alt.renderers.enable('default')" ] }, { "cell_type": "code", "execution_count": 9, "id": "af906f0c", "metadata": {}, "outputs": [], "source": [ "data_dir = \"/data/EMBER2024/\" # change this to where you unzipped the download" ] }, { "cell_type": "code", "execution_count": 3, "id": "66713c94", "metadata": {}, "outputs": [], "source": [ "train_df, test_df, challenge_df = thrember.read_metadata(data_dir)" ] }, { "cell_type": "code", "execution_count": 10, "id": "0b72ccbf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(3232000, 16)\n" ] }, { "data": { "text/html": [ "\n", "\n", "
\n", "" ], "text/plain": [ "alt.Chart(...)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Add a 'week' column to the dataframe\n", "plotdf = pl.concat([train_df, test_df])\n", "start_date = pd.Timestamp(\"2023-09-24\")\n", "plotdf = plotdf.with_columns(\n", " pl.from_epoch(\"first_submission_date\", time_unit=\"s\").alias(\"first_submission_dt\")\n", ")\n", "plotdf = plotdf.with_columns(\n", " (\n", " (pl.col(\"first_submission_dt\") - pl.lit(start_date)).dt.total_days() // 7\n", " ).cast(pl.Int64).alias(\"week\")\n", ")\n", "\n", "print(plotdf.shape)\n", "\n", "# Plot file types across weeks\n", "gbdf = plotdf.group_by([\"file_type\", \"week\"]).agg(pl.len().alias(\"count\"))\n", "alt.Chart(gbdf).mark_bar().encode(\n", " alt.X('week:O', axis=alt.Axis(title='Week First Seen')),\n", " alt.Y('count:Q', axis=alt.Axis(title='File Type')),\n", " alt.Color('file_type:N', scale=alt.Scale(range=[\"#4c78a8\", \"#54a24b\", \"#f58518\", \"#88d27a\", \"#9ecae9\", \"#ffbf79\"]),\n", " legend=alt.Legend(values=[\"Win32\", \"Win64\", \"Dot_Net\", \"APK\", \"ELF\", \"PDF\"]))\n", ")" ] }, { "cell_type": "code", "execution_count": 5, "id": "501106eb", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (10, 2)
familycount
stru32
"berbew"174481
"wacatac"81478
"expiro"74339
"cosmu"53965
"xmrig"28903
"upatre"25296
"sfone"22177
"glupteba"21670
"grandoreiro"20551
"flystudio"18141
" ], "text/plain": [ "shape: (10, 2)\n", "┌─────────────┬────────┐\n", "│ family ┆ count │\n", "│ --- ┆ --- │\n", "│ str ┆ u32 │\n", "╞═════════════╪════════╡\n", "│ berbew ┆ 174481 │\n", "│ wacatac ┆ 81478 │\n", "│ expiro ┆ 74339 │\n", "│ cosmu ┆ 53965 │\n", "│ xmrig ┆ 28903 │\n", "│ upatre ┆ 25296 │\n", "│ sfone ┆ 22177 │\n", "│ glupteba ┆ 21670 │\n", "│ grandoreiro ┆ 20551 │\n", "│ flystudio ┆ 18141 │\n", "└─────────────┴────────┘" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Get number of occurrences of each family\n", "family_counts = plotdf.select(\n", " pl.col(\"family\").value_counts().alias(\"family_counts\")\n", ").unnest(\"family_counts\")\n", "family_counts = (\n", " plotdf.filter(pl.col(\"family\").is_not_null())\n", " .select(pl.col(\"family\").value_counts())\n", " .unnest(\"family\")\n", " .sort(\"count\", descending=True)\n", ")\n", "family_counts.head(10)" ] }, { "cell_type": "code", "execution_count": 7, "id": "d6ab49b7", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "shape: (10, 2)
behaviorcount
stru32
"backdoor"228349
"virus"121971
"worm"76115
"downloader"61523
"spyware"55780
"coinminer"37680
"dropper"33291
"adware"24867
"phishing"21782
"ransom"16279
" ], "text/plain": [ "shape: (10, 2)\n", "┌────────────┬────────┐\n", "│ behavior ┆ count │\n", "│ --- ┆ --- │\n", "│ str ┆ u32 │\n", "╞════════════╪════════╡\n", "│ backdoor ┆ 228349 │\n", "│ virus ┆ 121971 │\n", "│ worm ┆ 76115 │\n", "│ downloader ┆ 61523 │\n", "│ spyware ┆ 55780 │\n", "│ coinminer ┆ 37680 │\n", "│ dropper ┆ 33291 │\n", "│ adware ┆ 24867 │\n", "│ phishing ┆ 21782 │\n", "│ ransom ┆ 16279 │\n", "└────────────┴────────┘" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Get number of occurrences of each behavior tag\n", "plotdf_explode = plotdf.filter(pl.col(\"behavior\").list.len() > 0).explode(\"behavior\")\n", "behavior_counts = (plotdf_explode.group_by(\"behavior\").agg(pl.len().alias(\"count\")).sort(\"count\", descending=True))\n", "behavior_counts.head(10)" ] }, { "cell_type": "code", "execution_count": null, "id": "935c3c7b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }