diff --git a/notebooks/train_and_publish_phishing_email_model.ipynb b/notebooks/train_and_publish_phishing_email_model.ipynb index b78a41adf2..9dba204ab6 100644 --- a/notebooks/train_and_publish_phishing_email_model.ipynb +++ b/notebooks/train_and_publish_phishing_email_model.ipynb @@ -10,7 +10,15 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T01:24:46.501377Z", + "iopub.status.busy": "2020-10-22T01:24:46.501109Z", + "iopub.status.idle": "2020-10-22T01:24:48.231489Z", + "shell.execute_reply": "2020-10-22T01:24:48.230823Z", + "shell.execute_reply.started": "2020-10-22T01:24:46.501346Z" + } + }, "outputs": [], "source": [ "import numpy as np\n", @@ -19,29 +27,57 @@ "from tensorflow.keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n", "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.callbacks import EarlyStopping\n", - "from tensorflow.keras import metrics" + "from tensorflow.keras import metrics\n", + "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T01:24:48.232814Z", + "iopub.status.busy": "2020-10-22T01:24:48.232604Z", + "iopub.status.idle": "2020-10-22T01:24:48.236001Z", + "shell.execute_reply": "2020-10-22T01:24:48.235405Z", + "shell.execute_reply.started": "2020-10-22T01:24:48.232794Z" + } + }, "outputs": [], "source": [ "maxLen = 128\n", "epochs = 50\n", "dim_embedding = 50\n", - "batch_size = 256\n", + "batch_size = 4\n", "dropout_rate = 0.25\n", "num_LSTM_cell = 64\n", - "trainDataFileName = 's3://smle-experiments/datasets/phishing_email/train.json'" + "trainDataFileName = 's3://smle-experiments/datasets/phishing_email/splunk_train.json'" ] }, { "cell_type": "code", "execution_count": 3, - "metadata": {}, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T01:24:48.237431Z", + "iopub.status.busy": "2020-10-22T01:24:48.237138Z", + "iopub.status.idle": "2020-10-22T01:25:04.369994Z", + "shell.execute_reply": "2020-10-22T01:25:04.369189Z", + "shell.execute_reply.started": "2020-10-22T01:24:48.237406Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.7/site-packages/dateutil/parser/_parser.py:1218: UnknownTimezoneWarning: tzname PST identified but not understood. Pass `tzinfos` argument in order to correctly return a timezone-aware datetime. In a future version, this will raise an exception.\n", + " category=UnknownTimezoneWarning)\n", + "/opt/conda/lib/python3.7/site-packages/dateutil/parser/_parser.py:1218: UnknownTimezoneWarning: tzname EDT identified but not understood. Pass `tzinfos` argument in order to correctly return a timezone-aware datetime. In a future version, this will raise an exception.\n", + " category=UnknownTimezoneWarning)\n" + ] + } + ], "source": [ "xTrain = []\n", "yTrain = []\n", @@ -70,82 +106,58 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T01:25:04.371518Z", + "iopub.status.busy": "2020-10-22T01:25:04.371269Z", + "iopub.status.idle": "2020-10-22T06:15:15.871435Z", + "shell.execute_reply": "2020-10-22T06:15:15.870561Z", + "shell.execute_reply.started": "2020-10-22T01:25:04.371493Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "If using Keras pass *_constraint arguments to layers.\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.where in 2.0, which has the same broadcast rule as np.where\n", - "Train on 40000 samples, validate on 10000 samples\n", "Epoch 1/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.1514 - acc: 0.9740 - val_loss: 0.0912 - val_acc: 0.9817\n", + "10103/10103 [==============================] - 892s 88ms/step - loss: 0.1135 - accuracy: 0.9708 - val_loss: 0.0803 - val_accuracy: 0.9725\n", "Epoch 2/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0990 - acc: 0.9796 - val_loss: 0.0904 - val_acc: 0.9817\n", + "10103/10103 [==============================] - 907s 90ms/step - loss: 0.0342 - accuracy: 0.9906 - val_loss: 0.0199 - val_accuracy: 0.9951\n", "Epoch 3/50\n", - "40000/40000 [==============================] - 43s 1ms/sample - loss: 0.0884 - acc: 0.9796 - val_loss: 0.0726 - val_acc: 0.9828\n", + "10103/10103 [==============================] - 898s 89ms/step - loss: 0.0142 - accuracy: 0.9965 - val_loss: 0.0098 - val_accuracy: 0.9977\n", "Epoch 4/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0726 - acc: 0.9798 - val_loss: 0.0640 - val_acc: 0.9830\n", + "10103/10103 [==============================] - 895s 89ms/step - loss: 0.0089 - accuracy: 0.9974 - val_loss: 0.0056 - val_accuracy: 0.9984\n", "Epoch 5/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0667 - acc: 0.9812 - val_loss: 0.0621 - val_acc: 0.9855\n", + "10103/10103 [==============================] - 910s 90ms/step - loss: 0.0061 - accuracy: 0.9982 - val_loss: 0.0033 - val_accuracy: 0.9992\n", "Epoch 6/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0665 - acc: 0.9813 - val_loss: 0.0598 - val_acc: 0.9849\n", + "10103/10103 [==============================] - 925s 92ms/step - loss: 0.0045 - accuracy: 0.9987 - val_loss: 0.0031 - val_accuracy: 0.9993\n", "Epoch 7/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0651 - acc: 0.9816 - val_loss: 0.0529 - val_acc: 0.9824\n", + "10103/10103 [==============================] - 911s 90ms/step - loss: 0.0039 - accuracy: 0.9987 - val_loss: 0.0052 - val_accuracy: 0.9988\n", "Epoch 8/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0532 - acc: 0.9830 - val_loss: 0.0393 - val_acc: 0.9889\n", + "10103/10103 [==============================] - 864s 85ms/step - loss: 0.0039 - accuracy: 0.9987 - val_loss: 0.0027 - val_accuracy: 0.9993\n", "Epoch 9/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0474 - acc: 0.9844 - val_loss: 0.0428 - val_acc: 0.9891\n", + "10103/10103 [==============================] - 947s 94ms/step - loss: 0.0030 - accuracy: 0.9992 - val_loss: 0.0018 - val_accuracy: 0.9997\n", "Epoch 10/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0419 - acc: 0.9891 - val_loss: 0.0359 - val_acc: 0.9912\n", + "10103/10103 [==============================] - 895s 89ms/step - loss: 0.0035 - accuracy: 0.9988 - val_loss: 0.0025 - val_accuracy: 0.9994\n", "Epoch 11/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0385 - acc: 0.9886 - val_loss: 0.0271 - val_acc: 0.9922\n", + "10103/10103 [==============================] - 891s 88ms/step - loss: 0.0031 - accuracy: 0.9992 - val_loss: 0.0035 - val_accuracy: 0.9993\n", "Epoch 12/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0369 - acc: 0.9894 - val_loss: 0.0251 - val_acc: 0.9937\n", + "10103/10103 [==============================] - 921s 91ms/step - loss: 0.0030 - accuracy: 0.9993 - val_loss: 0.0014 - val_accuracy: 0.9998\n", "Epoch 13/50\n", - "40000/40000 [==============================] - 46s 1ms/sample - loss: 0.0299 - acc: 0.9921 - val_loss: 0.0255 - val_acc: 0.9933\n", + "10103/10103 [==============================] - 967s 96ms/step - loss: 0.0025 - accuracy: 0.9993 - val_loss: 0.0037 - val_accuracy: 0.9992\n", "Epoch 14/50\n", - "40000/40000 [==============================] - 46s 1ms/sample - loss: 0.0242 - acc: 0.9942 - val_loss: 0.0181 - val_acc: 0.9948\n", + "10103/10103 [==============================] - 914s 90ms/step - loss: 0.0021 - accuracy: 0.9994 - val_loss: 0.0020 - val_accuracy: 0.9995\n", "Epoch 15/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0210 - acc: 0.9958 - val_loss: 0.0195 - val_acc: 0.9957\n", + "10103/10103 [==============================] - 907s 90ms/step - loss: 0.0030 - accuracy: 0.9992 - val_loss: 0.0027 - val_accuracy: 0.9993\n", "Epoch 16/50\n", - "40000/40000 [==============================] - 44s 1ms/sample - loss: 0.0179 - acc: 0.9961 - val_loss: 0.0146 - val_acc: 0.9965\n", + "10103/10103 [==============================] - 902s 89ms/step - loss: 0.0017 - accuracy: 0.9995 - val_loss: 0.0025 - val_accuracy: 0.9992\n", "Epoch 17/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0168 - acc: 0.9964 - val_loss: 0.0147 - val_acc: 0.9969\n", + "10103/10103 [==============================] - 868s 86ms/step - loss: 0.0022 - accuracy: 0.9995 - val_loss: 0.0026 - val_accuracy: 0.9996\n", "Epoch 18/50\n", - "40000/40000 [==============================] - 43s 1ms/sample - loss: 0.0179 - acc: 0.9959 - val_loss: 0.0125 - val_acc: 0.9968\n", + "10103/10103 [==============================] - 1004s 99ms/step - loss: 0.0023 - accuracy: 0.9994 - val_loss: 0.0023 - val_accuracy: 0.9996\n", "Epoch 19/50\n", - "40000/40000 [==============================] - 46s 1ms/sample - loss: 0.0262 - acc: 0.9909 - val_loss: 0.0168 - val_acc: 0.9971\n", - "Epoch 20/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0169 - acc: 0.9961 - val_loss: 0.0128 - val_acc: 0.9971\n", - "Epoch 21/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0163 - acc: 0.9962 - val_loss: 0.0134 - val_acc: 0.9966\n", - "Epoch 22/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0165 - acc: 0.9966 - val_loss: 0.0121 - val_acc: 0.9971\n", - "Epoch 23/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0143 - acc: 0.9967 - val_loss: 0.0115 - val_acc: 0.9970\n", - "Epoch 24/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0266 - acc: 0.9909 - val_loss: 0.0170 - val_acc: 0.9971\n", - "Epoch 25/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0158 - acc: 0.9964 - val_loss: 0.0119 - val_acc: 0.9971\n", - "Epoch 26/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0190 - acc: 0.9951 - val_loss: 0.0146 - val_acc: 0.9959\n", - "Epoch 27/50\n", - "40000/40000 [==============================] - 46s 1ms/sample - loss: 0.0173 - acc: 0.9956 - val_loss: 0.0134 - val_acc: 0.9973\n", - "Epoch 28/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0120 - acc: 0.9970 - val_loss: 0.0130 - val_acc: 0.9972\n", - "Epoch 29/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0125 - acc: 0.9970 - val_loss: 0.0123 - val_acc: 0.9973\n", - "Epoch 30/50\n", - "40000/40000 [==============================] - 45s 1ms/sample - loss: 0.0121 - acc: 0.9970 - val_loss: 0.0127 - val_acc: 0.9969\n" + "10103/10103 [==============================] - 991s 98ms/step - loss: 0.0018 - accuracy: 0.9996 - val_loss: 0.0040 - val_accuracy: 0.9994\n" ] } ], @@ -170,7 +182,15 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T06:15:15.873002Z", + "iopub.status.busy": "2020-10-22T06:15:15.872722Z", + "iopub.status.idle": "2020-10-22T06:15:16.579503Z", + "shell.execute_reply": "2020-10-22T06:15:16.578672Z", + "shell.execute_reply.started": "2020-10-22T06:15:15.872971Z" + } + }, "outputs": [], "source": [ "import os\n", @@ -199,51 +219,27 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T06:15:16.580642Z", + "iopub.status.busy": "2020-10-22T06:15:16.580410Z", + "iopub.status.idle": "2020-10-22T06:15:25.953086Z", + "shell.execute_reply": "2020-10-22T06:15:25.952109Z", + "shell.execute_reply.started": "2020-10-22T06:15:16.580616Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/smle/onnx.py:88: export_saved_model (from tensorflow.python.keras.saving.saved_model_experimental) is deprecated and will be removed in a future version.\n", + "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", - "Please use `model.save(..., save_format=\"tf\")` or `tf.keras.models.save_model(..., save_format=\"tf\")`.\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n", + "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:253: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n", - "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Predict: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Train: ['train']\n", - "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n", - "WARNING:tensorflow:Export includes no default signature!\n", - "INFO:tensorflow:No assets to save.\n", - "INFO:tensorflow:No assets to write.\n", - "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Predict: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval']\n", - "WARNING:tensorflow:Export includes no default signature!\n", - "INFO:tensorflow:No assets to save.\n", - "INFO:tensorflow:No assets to write.\n", - "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n", - "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n", - "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n", - "INFO:tensorflow:No assets to save.\n", - "INFO:tensorflow:No assets to write.\n", - "INFO:tensorflow:SavedModel written to: /tmp/phishing_email/saved_model.pb\n" + "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n", + "INFO:tensorflow:Assets written to: /tmp/phishing_email/assets\n" ] } ], @@ -253,103 +249,129 @@ "\n", "!rm -rf /tmp/{model_name}\n", "sample_data = pd.read_csv('s3://smle-experiments/datasets/phishing_email/sample_file.csv')\n", - "smle_context.publish(model, model_name=model_name, path = model_path, sample= sample_data)" + "smle_context.publish(model, model_name=model_name, path = model_path, sample= sample_data, input_names=[\"ml_in\"], output_names=[\"ml_out\"], tf2onnx=[\"--opset\", \"10\"])" ] }, { "cell_type": "code", "execution_count": 7, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T06:15:25.954666Z", + "iopub.status.busy": "2020-10-22T06:15:25.954322Z", + "iopub.status.idle": "2020-10-22T06:15:25.972751Z", + "shell.execute_reply": "2020-10-22T06:15:25.971808Z", + "shell.execute_reply.started": "2020-10-22T06:15:25.954638Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Metadata:\n", - "{'inputFields': [{'name': 'embedding_input:0',\n", - " 'size': 128,\n", - " 'type': 'floatTensor'}],\n", - " 'modelName': 'phishing_email',\n", - " 'outputFields': [{'name': 'dense/Sigmoid:0',\n", - " 'size': 1,\n", - " 'type': 'floatTensor'}]}\n", - "\n", + " {\n", + " \"modelName\": \"phishing_email\",\n", + " \"inputFields\": [\n", + " {\n", + " \"name\": \"ml_in\",\n", + " \"type\": \"floatTensor\",\n", + " \"size\": 128\n", + " }\n", + " ],\n", + " \"outputFields\": [\n", + " {\n", + " \"name\": \"ml_out\",\n", + " \"type\": \"floatTensor\",\n", + " \"size\": 1\n", + " }\n", + " ],\n", + " \"smle\": {\n", + " \"modelType\": \"tensorflow\",\n", + " \"published\": \"2020-10-22T06:15:25.712925+00:00\"\n", + " }\n", + "}\n", "ONNX model specs:\n", - "{'inputs': [{'name': 'embedding_input:0',\n", - " 'shape': ['unk__236', 128],\n", - " 'type': 'tensor(float)'}],\n", - " 'outputs': [{'name': 'dense/Sigmoid:0',\n", - " 'shape': ['unk__237', 1],\n", - " 'type': 'tensor(float)'}]}\n" + " {\n", + " \"Inputs\": [\n", + " {\n", + " \"name\": \"ml_in\",\n", + " \"type\": \"tensor(float)\",\n", + " \"shape\": [\n", + " \"unk__152\",\n", + " 128\n", + " ]\n", + " }\n", + " ],\n", + " \"Outputs\": [\n", + " {\n", + " \"name\": \"ml_out\",\n", + " \"type\": \"tensor(float)\",\n", + " \"shape\": [\n", + " \"unk__153\",\n", + " 1\n", + " ]\n", + " }\n", + " ]\n", + "}\n" ] } ], "source": [ - "import json\n", - "import onnxruntime as rt\n", - "from pprint import pprint\n", - "\n", - "\n", - "def inspect_model(dir_path, model_name):\n", - "\n", - " metadata = json.load(open(dir_path + \"metadata.json\"))\n", - " \n", - " print(\"Metadata:\")\n", - " pprint(metadata)\n", - " print()\n", - " \n", - " model_path = dir_path + model_name + \".onnx\"\n", - " sess = rt.InferenceSession(model_path)\n", - "\n", - " onnx_inputs = sess.get_inputs()\n", - " onnx_outputs = sess.get_outputs()\n", - "\n", - " inputs = [{\"name\": node.name, \"type\": node.type, \"shape\": node.shape} for node in onnx_inputs]\n", - " outputs = [{\"name\": node.name, \"type\": node.type, \"shape\": node.shape} for node in onnx_outputs]\n", - " onnx_model_specs = {\"inputs\": inputs, \"outputs\": outputs}\n", - " print(\"ONNX model specs:\")\n", - " \n", - " pprint(onnx_model_specs)\n", - "\n", - "\n", - "inspect_model(\"/tmp/\", model_name)" + "model_dir = \"/tmp/{:s}\".format(model_name)\n", + "smle_context.inspect(model_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SPL2 apply model for inference" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T06:15:25.974750Z", + "iopub.status.busy": "2020-10-22T06:15:25.974494Z", + "iopub.status.idle": "2020-10-22T06:15:25.979658Z", + "shell.execute_reply": "2020-10-22T06:15:25.978797Z", + "shell.execute_reply.started": "2020-10-22T06:15:25.974725Z" + } + }, "outputs": [], "source": [ "%%spl2_add_params from_python\n", "phishing_email = dict(\n", " model_path = \"s3://smle-experiments/models/phishing_email\",\n", " model_name = \"phishing_email\",\n", - " input_field = \"embedding_input:0\",\n", ")" ] }, { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2020-10-22T06:15:25.981856Z", + "iopub.status.busy": "2020-10-22T06:15:25.981612Z", + "iopub.status.idle": "2020-10-22T06:16:09.089904Z", + "shell.execute_reply": "2020-10-22T06:16:09.089171Z", + "shell.execute_reply.started": "2020-10-22T06:15:25.981833Z" + } + }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6aa0cfe6342a4e59b0ff9a68f1de5596", + "model_id": "d1832a412825443b9b887e42ac417efd", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))" + "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))" ] }, "metadata": {}, @@ -383,86 +405,135 @@ " \n", " \n", " \n", - " eventLine\n", + " end_time\n", + " start_time\n", + " body\n", + " entities\n", " probability\n", " \n", " \n", " \n", " \n", " 0\n", - " karem ahmed <karemahmed-18@hotmail.fr> PLEA...\n", - " [0.9720325]\n", + " Thu, 14 Mar 2002 14:34:48 -0800 (PST)\n", + " Thu, 14 Mar 2002 14:34:48 -0800 (PST)\n", + " TBD\n", + " TBD\n", + " 0.999014\n", " \n", " \n", " 1\n", - " nkomo robert <nkomo002@5fm.za.com> FAMILY R...\n", - " [0.9768889]\n", + " Mon, 21 May 2001 03:27:00 -0700 (PDT)\n", + " Mon, 21 May 2001 03:27:00 -0700 (PDT)\n", + " TBD\n", + " TBD\n", + " 0.999733\n", " \n", " \n", " 2\n", - " Mrs Mariam Taylor <familyboxjanetfamilybo...\n", - " [0.9780622999999999]\n", + " Fri, 4 May 2001 14:48:00 -0700 (PDT)\n", + " Fri, 4 May 2001 14:48:00 -0700 (PDT)\n", + " TBD\n", + " TBD\n", + " 0.999192\n", " \n", " \n", " 3\n", - " Mr.Fred Chima <fredi@fastermail.com> busi...\n", - " [0.97669697]\n", + " Thu, 1 Nov 2018 17:20:06 +0000\n", + " Thu, 1 Nov 2018 17:20:06 +0000\n", + " TBD\n", + " TBD\n", + " 0.998415\n", " \n", " \n", " 4\n", - " Alexander Afadia <eeaesq@123.com> Please repl...\n", - " [0.9775347999999999]\n", + " Wed, 23 Aug 2000 05:12:00 -0700 (PDT)\n", + " Wed, 23 Aug 2000 05:12:00 -0700 (PDT)\n", + " TBD\n", + " TBD\n", + " 0.999215\n", " \n", " \n", " ...\n", " ...\n", " ...\n", + " ...\n", + " ...\n", + " ...\n", " \n", " \n", - " 82\n", - " Comfort Somba. <comfort_somba11@yahoo.fr> F...\n", - " [0.97327405]\n", + " 218\n", + " Wednesday, 9 May 2018 at 07:34\n", + " Wednesday, 9 May 2018 at 07:34\n", + " TBD\n", + " TBD\n", + " 0.999302\n", " \n", " \n", - " 83\n", - " bintu pat <pat_bintu004@hotmail.com> TRUSTI...\n", - " [0.96824765]\n", + " 219\n", + " Mon, 25 Jun 2018 21:42:28 +0200\n", + " Mon, 25 Jun 2018 21:42:28 +0200\n", + " TBD\n", + " TBD\n", + " 0.999290\n", " \n", " \n", - " 84\n", - " FR RICHARD DAVID <unofice@katamail.com> FRO...\n", - " [0.96306074]\n", + " 220\n", + " \n", + " \n", + " TBD\n", + " TBD\n", + " 0.987345\n", " \n", " \n", - " 85\n", - " <joseph_m333@micasilla.net> Farmers Greetings...\n", - " [0.97275084]\n", + " 221\n", + " Fri, 11 May 2001 03:38:00 -0700 (PDT)\n", + " Fri, 11 May 2001 03:38:00 -0700 (PDT)\n", + " TBD\n", + " TBD\n", + " 0.999448\n", " \n", " \n", - " 86\n", - " MRS. LARISA SOSNITSKAYA <larisasosnkayapawou...\n", - " [0.9751048999999999]\n", + " 222\n", + " Mon, 26 Nov 2001 13:48:17 -0800 (PST)\n", + " Mon, 26 Nov 2001 13:48:17 -0800 (PST)\n", + " TBD\n", + " TBD\n", + " 0.999578\n", " \n", " \n", "\n", - "

87 rows × 2 columns

\n", + "

223 rows × 5 columns

\n", "" ], "text/plain": [ - " eventLine probability\n", - "0 karem ahmed PLEA... [0.9720325]\n", - "1 nkomo robert FAMILY R... [0.9768889]\n", - "2 Mrs Mariam Taylor busi... [0.97669697]\n", - "4 Alexander Afadia Please repl... [0.9775347999999999]\n", - ".. ... ...\n", - "82 Comfort Somba. F... [0.97327405]\n", - "83 bintu pat TRUSTI... [0.96824765]\n", - "84 FR RICHARD DAVID FRO... [0.96306074]\n", - "85 Farmers Greetings... [0.97275084]\n", - "86 MRS. LARISA SOSNITSKAYA " + "" ] }, "execution_count": 9, @@ -488,48 +559,82 @@ ], "source": [ "%%spl2 -q phishing_email\n", - "| from read_json(\"s3://smle-experiments/datasets/phishing_email/test.json\")\n", + "| from read_json(\"s3://smle-experiments/datasets/phishing_email/splunk_test.json\")\n", "| eval eventLine=concat(From, \" \", Subject, \" \", Content, \" \", \" \")\n", + "| where eventLine IS NOT NULL\n", "| eval mapC = {\" \":32,\"!\":33,\"\\\"\":34,\"#\":35,\"$$\":36,\"%\":37,\"&\":38,\"'\":39,\"(\":40,\")\":41,\"*\":42,\"+\":43,\",\":44,\"-\":45,\".\":46,\"/\":47,\"0\":48,\"1\":49,\"2\":50,\"3\":51,\"4\":52,\"5\":53,\"6\":54,\"7\":55,\"8\":56,\"9\":57,\":\":58,\";\":59,\"<\":60,\"=\":61,\">\":62,\"?\":63,\"@\":64,\"A\":65,\"B\":66,\"C\":67,\"D\":68,\"E\":69,\"F\":70,\"G\":71,\"H\":72,\"I\":73,\"J\":74,\"K\":75,\"L\":76,\"M\":77,\"N\":78,\"O\":79,\"P\":80,\"Q\":81,\"R\":82,\"S\":83,\"T\":84,\"U\":85,\"V\":86,\"W\":87,\"X\":88,\"Y\":89,\"Z\":90,\"[\":91,\"\\\\\":92,\"]\":93,\"^\":94,\"_\":95,\"`\":96,\"a\":97,\"b\":98,\"c\":99,\"d\":100,\"e\":101,\"f\":102,\"g\":103,\"h\":104,\"i\":105,\"j\":106,\"k\":107,\"l\":108,\"m\":109,\"n\":110,\"o\":111,\"p\":112,\"q\":113,\"r\":114,\"s\":115,\"t\":116,\"u\":117,\"v\":118,\"w\":119,\"x\":120,\"y\":121,\"z\":122,\"{\":123,\"|\":124,\"}\":125,\"~\":126}\n", - "| eval 'embedding_input:0' = for_each(\n", + "| eval ml_in = for_each(\n", " iterator(mvrange(1,129), \"i\"),\n", " cast(map_get(mapC, substr(eventLine, i, 1)), \"float\") )\n", - "| apply_model connection_id=\"\" path=\"$model_path\" name=\"$model_name\" \n", - "| rename 'dense/Sigmoid:0' AS probability \n", - "| where mvindex(probability, 0) > 0.5 \n", - "| select eventLine, probability \n", + "| apply_model connection_id=\"\" path=\"$model_path\" name=\"$model_name\"\n", + "| eval probability = mvindex(ml_out, 0) \n", + "| where probability > 0.5\n", + "| eval start_time = Date, end_time = Date, entities = \"TBD\", body = \"TBD\"\n", + "| select probability, body, entities, start_time, end_time\n", ";" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Note the versions of SMLE and Tensorflow used" + ] + }, { "cell_type": "code", "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "df = _.df" - ] - }, - { - "cell_type": "code", - "execution_count": 11, "metadata": { - "scrolled": true + "execution": { + "iopub.execute_input": "2020-10-22T06:16:09.091281Z", + "iopub.status.busy": "2020-10-22T06:16:09.091048Z", + "iopub.status.idle": "2020-10-22T06:16:10.815332Z", + "shell.execute_reply": "2020-10-22T06:16:10.814298Z", + "shell.execute_reply.started": "2020-10-22T06:16:09.091259Z" + } }, "outputs": [ { - "data": { - "text/plain": [ - "\" karem ahmed PLEASE VERY URGENT. FROM THE DESK OF Mr,KAREM AHMED.BILL AND EXCHANGE MANAGER,BANK OF AFRICA (B.O.A) OUAGADOUGOU,BURKINA FASO WEST AFRICA.PHONE CONTACT 00226.78.89.77.34DEAR FRIEND,I know you may be surprise to recieve this e-mail ; I got your contact address from the internet while I was searching for my friend that has similar name with you.I am the manager of bill and exchange BANK OF AFRICA (B.O.A) at foreign remittance department.There is a business I would want you to champion for me, in my department I discovered an abandoned sum of ($31.500.000 U.S) thirty one million five hundred thousands US dollars)In an account that belongs to one of our foreign customer who died along with his entire family on 25TH JULY, 2000 CONCORDE PLANE CRASH [Flight AF4590] with the whole passengers aboard. The name of the deceased man was(MR.ANDREAS SCHRANNER from Munich Germany)N.B. In other for you to believe me honestly, visit the web site to enable you know whether we can work together, below is the website. http://news.bbc.co.uk/1/hi/world/europe/859479.stmYou have to understand that I come crossed this huge amount of money when I was arranging the departmental customers file to submit to the bank management for the annual audit of the year.Since we got information about his death, we have been expecting his next of kin to come over and claim his money because we cannot release it unless somebody applies for it as next of kin or relation to the deceased as indicated in our banking guidelines, but unfortunately we learnt that all his supposed next of kin's or relation died alongside with him at the plane crash leaving nobody behind for the claim.It is therefore upon this discovery that I now decided to make this business proposal to you and release the money to you as the next of kin or relation to the deceased for safety and subsequent disbursement since nobody is coming for it and I dont want this money to go into the Bank treasury as unclaimed Bill.The Banking law and guideline here stipulates that if such money remained unclaimed after some years, the money will be transferred into the Bank treasury as unclaimed fund.The request of foreigner as next of kin in this business is occasioned by the fact that the customer was a foreigner and a Burkina citizen cannot stand as next of kin to a foreigner.In fact I could have done this deal alone but because of my position in this country as a civil servant (A Banker), we are not allowed to operate any foreign account and would eventually raise an eye brow on my side during the time of transfer because I work in the same bank. This is the actual reason why it will require a second party or fellow who will forward claims as the next of kin to the Bank and also present a foreign account where he will need the money to be re-transferred into on his request as it may be after due verification and clarification by the correspondent branch of the bank where the whole money will be remitted from to your own designation bank account.I dont want this money to go into the Bank treasury as unclaimed Bill, I agree that 30 % of this money will be for you as foreign partner, in respect to the provision of a foreign account, 10 % will be set aside for expenses incurred during the business and 60 % would be for me. There after I will visit your country for disbursement according to the percentages indicated. Therefore to enable immediate transfer of this fund to you as arranged, you must apply first to the bank as relations or next of kin of the deceased indicating your bank name, your bank account number, your private telephone and fax number for easy and effective communication and location where in the money will be remitted.All modalities of this transaction have been carefully worked out and once started will not take more than fourteen (14) working days, with your full support.This transaction is 100% risk free. it involve no any implication and no any precaution there after.Upon receipt of your reply, I will send to you by fax or email the text of the application. I will not fail to bring to your notice that this transaction is hitch free and that you should not entertain any atom of fear as all required arrangements have been made for the transfer.You should contact me immediately as soon as you receive this letterreply to thi box. at;Trusting to hear from you immediately.Yours faithfully,Mr,KAREM AHMEDBill and exchange manager,BANK OF AFRICA_________________________________________________________________Personnalisez votre Messenger avec Live.com http://www.windowslive.fr/livecom/ \"" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: smle\n", + "Version: 0.2.12\n", + "Summary: Splunk Machine Learning Environment\n", + "Home-page: https://cd.splunkdev.com/ML/smle\n", + "Author: SplunkMLTeam\n", + "Author-email: mleng@splunk.com\n", + "License: Splunk Software License\n", + "Location: /opt/conda/lib/python3.7/site-packages\n", + "Requires: scipy, onnx, minio, packaging, toml, arrow, pandas, onnxruntime, python-json-logger, numpy, boto3\n", + "Required-by: \n", + "---\n", + "Name: tensorflow\n", + "Version: 2.3.0\n", + "Summary: TensorFlow is an open source machine learning framework for everyone.\n", + "Home-page: https://www.tensorflow.org/\n", + "Author: Google Inc.\n", + "Author-email: packages@tensorflow.org\n", + "License: Apache 2.0\n", + "Location: /opt/conda/lib/python3.7/site-packages\n", + "Requires: termcolor, keras-preprocessing, six, protobuf, google-pasta, astunparse, tensorflow-estimator, scipy, numpy, gast, opt-einsum, tensorboard, absl-py, wrapt, h5py, wheel, grpcio\n", + "Required-by: \n", + "---\n", + "Name: tf2onnx\n", + "Version: 1.6.3\n", + "Summary: Tensorflow to ONNX converter\n", + "Home-page: https://github.com/onnx/tensorflow-onnx\n", + "Author: onnx@microsoft.com\n", + "Author-email: onnx@microsoft.com\n", + "License: UNKNOWN\n", + "Location: /opt/conda/lib/python3.7/site-packages\n", + "Requires: numpy, onnx, requests, six\n", + "Required-by: \n" + ] } ], "source": [ - "df.iloc[0]['eventLine']" + "!pip show smle tensorflow tf2onnx" ] }, { @@ -556,7 +661,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.7.8" } }, "nbformat": 4,