admin/schwartz1375-genai-essentials
1
0
Fork 0
mirror of https://github.com/schwartz1375/genai-essentials synced 2026-08-09 13:08:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: stabilize Ollama 0.13.x embeddings by reducing chunk size and configuring num_ctx in RAG pipeline

Browse Source
This commit is contained in:
schwartz1375
2025-12-04 20:20:25 -05:00
parent a702fdb1a1
commit 563737641a
1 changed files with 59 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
local_rag.ipynb
+59 -34
View File
@@ -24,7 +24,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "2770431f",
"metadata": {},
"outputs": [],
@@ -40,7 +40,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 6,
"id": "1de79a00",
"metadata": {},
"outputs": [],
@@ -108,7 +108,10 @@
" loader_cls = loaders[loader_name]\n",
" pattern = glob_patterns[loader_cls]\n",
" loader = DirectoryLoader(\n",
" dir_path, glob=pattern, loader_cls=loader_cls, show_progress=True\n",
" dir_path,\n",
" glob=pattern,\n",
" loader_cls=loader_cls,\n",
" show_progress=True,\n",
" )\n",
" loaded = loader.load()\n",
" print(f\"📂 Loaded {len(loaded)} from {dir_path}\")\n",
@@ -121,9 +124,10 @@
"\n",
"# --- Text Splitting ---\n",
"def split_documents(docs):\n",
" # Smaller chunks to keep embeddings safe for Ollama 0.13.x\n",
" splitter = RecursiveCharacterTextSplitter(\n",
" chunk_size=1000,\n",
" chunk_overlap=200,\n",
" chunk_size=500, # was 1000\n",
" chunk_overlap=100, # was 200\n",
" separators=[\"\\n\\n\", \"\\n\", \".\", \"!\", \"?\", \",\", \" \", \"\"],\n",
" length_function=len,\n",
" )\n",
@@ -137,7 +141,9 @@
" assert_directory_writable(persist_dir)\n",
" chunks = split_documents(docs)\n",
" vectordb = Chroma.from_documents(\n",
" documents=chunks, embedding=embedding_model, persist_directory=persist_dir\n",
" documents=chunks,\n",
" embedding=embedding_model,\n",
" persist_directory=persist_dir,\n",
" )\n",
" print(f\"✅ VectorDB built at: {persist_dir}\")\n",
" return vectordb\n",
@@ -145,7 +151,10 @@
"\n",
"# --- Vector Store Load ---\n",
"def load_vectordb(persist_dir, embedding_model):\n",
" vectordb = Chroma(persist_directory=persist_dir, embedding_function=embedding_model)\n",
" vectordb = Chroma(\n",
" persist_directory=persist_dir,\n",
" embedding_function=embedding_model,\n",
" )\n",
" print(f\"✅ VectorDB loaded from: {persist_dir}\")\n",
" return vectordb\n",
"\n",
@@ -171,12 +180,18 @@
"\n",
"def create_qa_chain(vectordb):\n",
" try:\n",
" llm = OllamaLLM(model=LLM_MODEL)\n",
" llm = OllamaLLM(\n",
" model=LLM_MODEL,\n",
" base_url=\"http://localhost:11434\", # make endpoint explicit\n",
" )\n",
" # Test LLM model\n",
" llm.invoke(\"test\")\n",
" print(f\"🤖 LLM ready: {LLM_MODEL}\")\n",
" except Exception as e:\n",
" raise RuntimeError(f\"❌ LLM model '{LLM_MODEL}' not available. Run: ollama pull {LLM_MODEL}. Error: {e}\")\n",
" raise RuntimeError(\n",
" f\"❌ LLM model '{LLM_MODEL}' not available. \"\n",
" f\"Run: ollama pull {LLM_MODEL}. Error: {e}\"\n",
" )\n",
" return RetrievalQA.from_chain_type(\n",
" llm=llm,\n",
" retriever=vectordb.as_retriever(search_kwargs={\"k\": 5}),\n",
@@ -199,15 +214,23 @@
" RetrievalQA: A ready-to-use QA chain for question answering.\n",
" \"\"\"\n",
" print(\"🚀 Initializing pipeline...\")\n",
" \n",
" # Validate Ollama models are available\n",
"\n",
" # Validate Ollama embedding model is available\n",
" try:\n",
" embedding_model = OllamaEmbeddings(model=EMBEDDING_MODEL)\n",
" embedding_model = OllamaEmbeddings(\n",
" model=EMBEDDING_MODEL,\n",
" base_url=\"http://localhost:11434\",\n",
" num_ctx=2048, # keep context size modest and explicit\n",
" # keep_alive can be added as seconds (int) if you like, e.g. keep_alive=300\n",
" )\n",
" # Test embedding model\n",
" embedding_model.embed_query(\"test\")\n",
" print(f\"✅ Embedding model '{EMBEDDING_MODEL}' is ready\")\n",
" except Exception as e:\n",
" raise RuntimeError(f\"❌ Embedding model '{EMBEDDING_MODEL}' not available. Run: ollama pull {EMBEDDING_MODEL}. Error: {e}\")\n",
" raise RuntimeError(\n",
" f\"❌ Embedding model '{EMBEDDING_MODEL}' not available. \"\n",
" f\"Run: ollama pull {EMBEDDING_MODEL}. Error: {e}\"\n",
" )\n",
"\n",
" if build or not os.path.exists(os.path.join(CHROMA_DIR, \"chroma.sqlite3\")):\n",
" docs = load_documents(DATA_DIR, SUPPORTED_FORMATS)\n",
@@ -216,7 +239,7 @@
" vectordb = load_vectordb(CHROMA_DIR, embedding_model)\n",
"\n",
" qa_chain = create_qa_chain(vectordb)\n",
" return qa_chain"
" return qa_chain\n"
]
},
{
@@ -229,7 +252,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 7,
"id": "70fd9668",
"metadata": {},
"outputs": [
@@ -245,7 +268,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 20/20 [00:00<00:00, 20.36it/s]\n"
"100%|██████████| 20/20 [00:00<00:00, 20.93it/s]\n"
]
},
{
@@ -268,7 +291,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 8/8 [00:11<00:00, 1.50s/it]\n"
"100%|██████████| 8/8 [00:02<00:00, 2.91it/s]\n"
]
},
{
@@ -302,7 +325,7 @@
"📂 Loaded 0 from ./data/txt\n",
"📄 Total docs loaded: 414\n",
"✅ Confirmed write access to: ./chromadb_store\n",
"🔪 1730 chunks created\n",
"🔪 3400 chunks created\n",
"✅ VectorDB built at: ./chromadb_store\n",
"🤖 LLM ready: gemma3\n"
]
@@ -324,7 +347,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 8,
"id": "64058cb9",
"metadata": {},
"outputs": [
@@ -345,7 +368,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"I understand you're asking a question, but the provided context doesn't contain any information about pickles or why they might be bad. It focuses on a software product’s security."
"I don't know. The provided context discusses security aspects of software products, including Safe Browsing, Google Play Protect, and the absence of unsafe functions. It does not contain any information about pickles."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -360,7 +383,8 @@
"text": [
"\n",
"📄 Sources:\n",
"E02781980_Telecommunications_Security_CoP_Accessible.pdf, Page 120\n",
"Tutorial-4-MultipleConnections.md, Page N/A\n",
"guidance-mobile-communications-best-practices.pdf, Page 3\n",
"\n",
"❓ Question: What is a ClickFix?\n"
]
@@ -370,7 +394,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"The ClickFix tactic deceives users into downloading and running malware on their machines without them knowing. Threat actors initiate these campaigns by logging into websites with stolen credentials and installing fake plugins in compromised environments. Once installed, the plugins inject malicious JavaScript containing fake browser update malware that uses blockchain and smart contracts to obtain malicious payloads. When executed in the browser, JavaScript presents users with fake browser update notifications that guide them to install malware on their computer (usually remote access trojans and various infostealers like Vidar Stealer, DarkGate, and Lumma Stealer)."
"I don't know. The provided context discusses Singletons, Endianess, Locks, and Certificate Verification, but doesn’t contain information about ClickFix."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -385,7 +409,7 @@
"text": [
"\n",
"📄 Sources:\n",
"clickfix-attacks-sector-alert-tlpclear.pdf, Page 0\n",
"Tutorial-7-PythonNetworkingExpansion.md, Page N/A\n",
"\n",
"❓ Question: What is LummaC2?\n"
]
@@ -395,7 +419,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"LummaC2.exe is a file that, upon execution, enters a main routine with four sub-routines. The first routine decrypts strings for a message box displayed to the user."
"I don't know. The context provided only discusses Singletons, Endianess, Locks, and Certificate Verification. It does not contain information about LummaC2."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -410,7 +434,7 @@
"text": [
"\n",
"📄 Sources:\n",
"aa25-141b-threat-actors-deploy-lummac2-malware-to-exfiltrate-sensitive-data-from-organizations.pdf, Page 1\n"
"Tutorial-7-PythonNetworkingExpansion.md, Page N/A\n"
]
}
],
@@ -466,7 +490,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 9,
"id": "148d2858",
"metadata": {},
"outputs": [],
@@ -532,7 +556,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 10,
"id": "edfcf6b8",
"metadata": {},
"outputs": [
@@ -549,7 +573,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"The ClickFix tactic deceives users into downloading and running malware on their machines without realizing it. Threat actors initiate these campaigns by logging into websites with stolen credentials and installing fake plugins in compromised environments. Once installed, the plugins inject malicious JavaScript containing fake browser update malware that uses blockchain and smart contracts to obtain malicious payloads. When executed in the browser, JavaScript presents users with fake browser update notifications that guide them to install malware."
"I don't know. The provided context discusses Singletons, Endianess, Locks, and Certificate Verification, but doesn’t contain information about ClickFix."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -564,7 +588,7 @@
"text": [
"\n",
"📄 Sources:\n",
"clickfix-attacks-sector-alert-tlpclear.pdf, Page 0\n",
"Tutorial-7-PythonNetworkingExpansion.md, Page N/A\n",
"\n",
"❓ Question: What is LummaC2?\n"
]
@@ -574,7 +598,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"LummaC2.exe is a file that, upon execution, enters a main routine with four sub-routines. The first routine decrypts strings for a message box displayed to the user."
"I don't know. The provided context discusses Singletons, Endianess, Locks, and Certificate Verification – it doesn’t contain information about LummaC2."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -589,7 +613,7 @@
"text": [
"\n",
"📄 Sources:\n",
"aa25-141b-threat-actors-deploy-lummac2-malware-to-exfiltrate-sensitive-data-from-organizations.pdf, Page 1\n",
"Tutorial-7-PythonNetworkingExpansion.md, Page N/A\n",
"\n",
"❓ Question: Why is the sky blue?\n"
]
@@ -599,7 +623,7 @@
"text/markdown": [
"**Answer:**\n",
"\n",
"I do not know. The provided context discusses cyber threats, geopolitical monitoring, and security intelligence – it does not contain information about why the sky is blue."
"I don't know. The context provided discusses Sekoia, Singletons, Endianess, and Locks, none of which relate to the color of the sky."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
@@ -614,7 +638,8 @@
"text": [
"\n",
"📄 Sources:\n",
"CTA-RU-2024-0530.pdf, Page 2\n"
"Tutorial-4-MultipleConnections.md, Page N/A\n",
"clickfix-attacks-sector-alert-tlpclear.pdf, Page 1\n"
]
}
],
@@ -665,7 +690,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.11"
"version": "3.12.12"
}
},
"nbformat": 4,