diff --git a/cmd/jupyter/post_data.ipynb b/cmd/jupyter/post_data.ipynb new file mode 100644 index 0000000..13042f6 --- /dev/null +++ b/cmd/jupyter/post_data.ipynb @@ -0,0 +1,51 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "64597508-285a-43e7-a38b-976166c431da", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from pathlib import Path\n", + "\n", + "file_path = \"./sample_files/structured/process_data.json\"\n", + "json_data = Path(file_path).read_text()\n", + "\n", + "basic_auth_user = \"n\"\n", + "basic_auth_password = \"Qwerty12345\"\n", + "\n", + "NEMESIS_HTTP_SERVER=\"http://ingress-nginx-controller.ingress-nginx.svc.cluster.local/\"\n", + "\n", + "requests.post(f\"{NEMESIS_HTTP_SERVER}api/data\", data=json_data, auth=(basic_auth_user, basic_auth_password)).json()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.6 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dockerfiles/jupyter.Dockerfile b/dockerfiles/jupyter.Dockerfile new file mode 100644 index 0000000..23c3bf4 --- /dev/null +++ b/dockerfiles/jupyter.Dockerfile @@ -0,0 +1,9 @@ +FROM jupyter/datascience-notebook:latest + +# RUN pip install kafka-python + +WORKDIR /home/jovyan/ +RUN mv work examples +COPY cmd/jupyter/post_data.ipynb /home/jovyan/examples +COPY packages/protobufs/nemesis.proto /home/jovyan/examples +COPY sample_files/ examples/sample_files \ No newline at end of file diff --git a/kubernetes/jupyter/deployment.yaml b/kubernetes/jupyter/deployment.yaml new file mode 100644 index 0000000..70789ba --- /dev/null +++ b/kubernetes/jupyter/deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jupyter + labels: + app: jupyter +spec: + replicas: 1 + selector: + matchLabels: + app: jupyter + strategy: {} + template: + metadata: + labels: + app: jupyter + spec: + containers: + - env: + - name: JUPYTER_TOKEN + valueFrom: + secretKeyRef: + name: dashboard-creds + key: dashboard-password + + - name: JUPYTER_PORT + value: "8888" + - name: NOTEBOOK_ARGS + value: "--NotebookApp.base_url=/jupyter/" + image: nemesis-jupyter + name: nemesis-jupyter + ports: + - containerPort: 8888 + name: http + readinessProbe: + httpGet: + path: /jupyter/api + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 10 + + restartPolicy: Always +status: {} diff --git a/kubernetes/jupyter/ingress.yaml b/kubernetes/jupyter/ingress.yaml new file mode 100644 index 0000000..8ee9c11 --- /dev/null +++ b/kubernetes/jupyter/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: jupyter-ingress + annotations: + # TODO: Basic-auth breaks the websocket for some reasone + # nginx.ingress.kubernetes.io/auth-type: basic + # nginx.ingress.kubernetes.io/auth-secret: basic-auth + # nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/rewrite-target: /jupyter/$2 + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /jupyter(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: jupyter + port: + number: 8888 \ No newline at end of file diff --git a/kubernetes/jupyter/service.yaml b/kubernetes/jupyter/service.yaml new file mode 100644 index 0000000..7ce6da6 --- /dev/null +++ b/kubernetes/jupyter/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: jupyter + name: jupyter +spec: + type: NodePort + ports: + - name: "8888" + port: 8888 + targetPort: 8888 + nodePort: 31888 + selector: + app: jupyter +status: + loadBalancer: {} \ No newline at end of file diff --git a/skaffold.yaml b/skaffold.yaml index b81fd08..bd9c77d 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -166,18 +166,51 @@ portForward: # port: 5672 # localPort: 5672 # address: 0.0.0.0 -# - &portForward-gotenberg -# resourceType: service -# resourceName: gotenberg -# namespace: default -# port: 3000 -# address: 0.0.0.0 -# - &portForward-tika -# resourceType: service -# resourceName: tika -# namespace: default -# port: 9998 -# address: 0.0.0.0 + +--- +############################################################################# +# +# Jupyter - Deploys an instance of Jupyter Notebooks +# +# Usage: skaffold dev -f skaffold.yaml -m jupyter -p jupyter +############################################################################# +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: jupyter +build: + tagPolicy: + inputDigest: {} + local: + push: false + tryImportMissing: true + useBuildkit: true + concurrency: 0 + artifacts: + - &build-jupyter + image: nemesis-jupyter + docker: + dockerfile: dockerfiles/jupyter.Dockerfile + +manifests: + rawYaml: [] + +deploy: + kubectl: {} + +profiles: + - name: jupyter + manifests: + rawYaml: + - kubernetes/jupyter/* + +portForward: + - &portForward-jupyter + resourceType: service + resourceName: jupyter + namespace: default + port: 8888 + address: 0.0.0.0 --- ######################################################################################### @@ -222,25 +255,6 @@ manifests: deploy: kubectl: {} -# portForward: -# - &portForward-dotnet -# resourceType: service -# resourceName: dotnet -# namespace: default -# port: 9800 -# address: 0.0.0.0 -# - &portForward-nlp -# resourceType: service -# resourceName: nlp -# namespace: default -# port: 9803 -# address: 0.0.0.0 -# - &portForward-tensorflow-serving -# resourceType: service -# resourceName: tensorflow-serving -# namespace: default -# port: 8501 -# address: 0.0.0.0 --- apiVersion: skaffold/v4beta6 kind: Config @@ -270,14 +284,6 @@ manifests: - &k8s-enrichment kubernetes/enrichment/* deploy: kubectl: {} - -# portForward: -# - &portForward-dotnet -# resourceType: service -# resourceName: enrichment-webapi -# namespace: default -# port: 9910 -# address: 0.0.0.0 --- apiVersion: skaffold/v4beta6 kind: Config