Compare commits

..

1 Commits

Author SHA1 Message Date
phernandez 615d8ba685 perf: Add batch processing for Postgres sync optimization
Implements streaming batch processing to reduce database roundtrips from 50K-80K to ~4K-6K for large projects (10K files).

**Phase 1: Scan Optimization**
- Add entity_repository.get_by_file_paths_batch() for bulk entity fetching
- Reduces scan phase from N queries to 1 batched query
- Impact: 427 files scanned with 2 queries vs 427 before

**Phase 2: Batch Infrastructure**
- Add sync_batch_size config (default: 100 files per batch)
- Add chunks() utility for streaming batch processing
- Add entity_repository.upsert_entities() for bulk inserts/updates
- Add observation_repository.delete_by_entity_ids() for batch deletes
- Add relation_repository.delete_outgoing_relations_from_entities() for batch deletes

**Phase 3: Sync Phase Optimization**
- Add sync_markdown_batch() method with 3-phase processing:
  1. Parse all files in batch (no DB operations)
  2. Bulk upsert entities in single transaction
  3. Post-process relations, checksums, search indexing per file
- Update new/modified file loops to use batch processing
- Add exception handling for circuit breaker and fatal errors
- Separate markdown/regular file processing in batches

**Test Updates**
- Update circuit breaker tests to work with batch architecture
- Change mocks from sync_markdown_file to sync_markdown_batch
- Update fatal error test to mock upsert_entities
- All circuit breaker tests passing (8/8)

**Expected Performance**
- Initial bulk import: ~10-15 queries/file (vs 43 before)
- Incremental sync: Massive scan improvement + batch upsert benefits
- Handles both new files and existing files efficiently

Addresses N+1 query patterns and transaction overhead with remote Postgres databases while maintaining circuit breaker functionality and proper error handling.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-20 21:16:04 -06:00
72 changed files with 704 additions and 5828 deletions
+1 -2
View File
@@ -52,5 +52,4 @@ ENV/
# claude action
claude-output
**/.claude/settings.local.json
.mcp.json
**/.claude/settings.local.json
-1
View File
@@ -264,6 +264,5 @@ With GitHub integration, the development workflow includes:
2. **Contribution tracking** - All of Claude's contributions are properly attributed in the Git history
3. **Branch management** - Claude can create feature branches for implementations
4. **Documentation maintenance** - Claude can keep documentation updated as the code evolves
5. **Code Commits**: ALWAYS sign off commits with `git commit -s`
This level of integration represents a new paradigm in AI-human collaboration, where the AI assistant becomes a full-fledged team member rather than just a tool for generating code snippets.
+4 -3
View File
@@ -15,6 +15,7 @@ dependencies = [
"aiosqlite>=0.20.0",
"greenlet>=3.1.1",
"pydantic[email,timezone]>=2.10.3",
"icecream>=2.1.3",
"mcp>=1.2.0",
"pydantic-settings>=2.6.1",
"loguru>=0.7.3",
@@ -34,9 +35,8 @@ dependencies = [
"python-dotenv>=1.1.0",
"pytest-aio>=1.9.0",
"aiofiles>=24.1.0", # Async file I/O
"logfire[fastapi]>=0.73.0", # Optional observability (disabled by default via config)
"logfire>=0.73.0", # Optional observability (disabled by default via config)
"asyncpg>=0.30.0",
"nest-asyncio>=1.6.0", # For Alembic migrations with Postgres
]
@@ -81,7 +81,8 @@ dev = [
"pytest-xdist>=3.0.0",
"ruff>=0.1.6",
"freezegun>=1.5.5",
"nest-asyncio>=1.6.0",
"psycopg2-binary>=2.9.0", # For Alembic migrations with Postgres
]
[tool.hatch.version]
+25 -91
View File
@@ -1,25 +1,14 @@
"""Alembic environment configuration."""
import asyncio
import os
from logging.config import fileConfig
# Allow nested event loops (needed for pytest-asyncio and other async contexts)
# Note: nest_asyncio doesn't work with uvloop, so we handle that case separately
try:
import nest_asyncio
nest_asyncio.apply()
except (ImportError, ValueError):
# nest_asyncio not available or can't patch this loop type (e.g., uvloop)
pass
from sqlalchemy import engine_from_config, pool
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
from basic_memory.config import ConfigManager
from basic_memory.config import ConfigManager, DatabaseBackend
# set config.env to "test" for pytest to prevent logging to file in utils.setup_logging()
os.environ["BASIC_MEMORY_ENV"] = "test"
@@ -46,6 +35,12 @@ if not current_url or current_url == "driver://user:pass@localhost/dbname":
sqlalchemy_url = DatabaseType.get_db_url(
app_config.database_path, DatabaseType.FILESYSTEM, app_config
)
# For Postgres, Alembic needs synchronous driver (psycopg2), not async (asyncpg)
if app_config.database_backend == DatabaseBackend.POSTGRES:
# Convert asyncpg URL to psycopg2 URL for Alembic
sqlalchemy_url = sqlalchemy_url.replace("postgresql+asyncpg://", "postgresql://")
config.set_main_option("sqlalchemy.url", sqlalchemy_url)
# Interpret the config file for Python logging.
@@ -90,89 +85,28 @@ def run_migrations_offline() -> None:
context.run_migrations()
def do_run_migrations(connection):
"""Execute migrations with the given connection."""
context.configure(
connection=connection,
target_metadata=target_metadata,
include_object=include_object,
render_as_batch=True,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations(connectable):
"""Run migrations asynchronously with AsyncEngine."""
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
Supports both sync engines (SQLite) and async engines (PostgreSQL with asyncpg).
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# Check if a connection/engine was provided (e.g., from run_migrations)
connectable = context.config.attributes.get("connection", None)
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
if connectable is None:
# No connection provided, create engine from config
url = context.config.get_main_option("sqlalchemy.url")
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
include_object=include_object,
render_as_batch=True,
)
# Check if it's an async URL (sqlite+aiosqlite or postgresql+asyncpg)
if url and ("+asyncpg" in url or "+aiosqlite" in url):
# Create async engine for asyncpg or aiosqlite
connectable = create_async_engine(
url,
poolclass=pool.NullPool,
future=True,
)
else:
# Create sync engine for regular sqlite or postgresql
connectable = engine_from_config(
context.config.get_section(context.config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
# Handle async engines (PostgreSQL with asyncpg)
if isinstance(connectable, AsyncEngine):
# Try to run async migrations
# nest_asyncio allows asyncio.run() from within event loops, but doesn't work with uvloop
try:
asyncio.run(run_async_migrations(connectable))
except RuntimeError as e:
if "cannot be called from a running event loop" in str(e):
# We're in a running event loop (likely uvloop) - need to use a different approach
# Create a new thread to run the async migrations
import concurrent.futures
def run_in_thread():
"""Run async migrations in a new event loop in a separate thread."""
new_loop = asyncio.new_event_loop()
asyncio.set_event_loop(new_loop)
try:
new_loop.run_until_complete(run_async_migrations(connectable))
finally:
new_loop.close()
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(run_in_thread)
future.result() # Wait for completion and re-raise any exceptions
else:
raise
else:
# Handle sync engines (SQLite) or sync connections
if hasattr(connectable, "connect"):
# It's an engine, get a connection
with connectable.connect() as connection:
do_run_migrations(connection)
else:
# It's already a connection
do_run_migrations(connectable)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
+5 -22
View File
@@ -20,16 +20,6 @@ from basic_memory.api.routers import (
search,
prompt_router,
)
from basic_memory.api.v2.routers import (
knowledge_router as v2_knowledge,
project_router as v2_project,
memory_router as v2_memory,
search_router as v2_search,
resource_router as v2_resource,
directory_router as v2_directory,
prompt_router as v2_prompt,
importer_router as v2_importer,
)
from basic_memory.config import ConfigManager
from basic_memory.services.initialization import initialize_file_sync, initialize_app
@@ -76,7 +66,8 @@ app = FastAPI(
lifespan=lifespan,
)
# Include v1 routers
# Include routers
app.include_router(knowledge.router, prefix="/{project}")
app.include_router(memory.router, prefix="/{project}")
app.include_router(resource.router, prefix="/{project}")
@@ -86,20 +77,12 @@ app.include_router(directory_router.router, prefix="/{project}")
app.include_router(prompt_router.router, prefix="/{project}")
app.include_router(importer_router.router, prefix="/{project}")
# Include v2 routers (ID-based paths)
app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
app.include_router(v2_search, prefix="/v2/projects/{project_id}")
app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
app.include_router(v2_project, prefix="/v2")
# Project resource router works across projects
# Project resource router works accross projects
app.include_router(project.project_resource_router)
app.include_router(management.router)
# Auth routes are handled by FastMCP automatically when auth is enabled
@app.exception_handler(Exception)
async def exception_handler(request, exc): # pragma: no cover
@@ -1,11 +1,4 @@
"""Router for knowledge graph operations.
⚠️ DEPRECATED: This v1 API is deprecated and will be removed on June 30, 2026.
Please migrate to /v2/{project}/knowledge endpoints which use entity IDs instead
of path-based identifiers for improved performance and stability.
Migration guide: See docs/migration/v1-to-v2.md
"""
"""Router for knowledge graph operations."""
from typing import Annotated
@@ -32,11 +25,7 @@ from basic_memory.schemas import (
from basic_memory.schemas.request import EditEntityRequest, MoveEntityRequest
from basic_memory.schemas.base import Permalink, Entity
router = APIRouter(
prefix="/knowledge",
tags=["knowledge"],
deprecated=True, # Marks entire router as deprecated in OpenAPI docs
)
router = APIRouter(prefix="/knowledge", tags=["knowledge"])
async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None:
+8 -50
View File
@@ -50,7 +50,6 @@ async def get_project(
) # pragma: no cover
return ProjectItem(
id=found_project.id,
name=found_project.name,
path=normalize_project_path(found_project.path),
is_default=found_project.is_default or False,
@@ -81,17 +80,9 @@ async def update_project(
raise HTTPException(status_code=400, detail="Path must be absolute")
# Get original project info for the response
old_project = await project_service.get_project(name)
if not old_project:
raise HTTPException(
status_code=400, detail=f"Project '{name}' not found in configuration"
)
old_project_info = ProjectItem(
id=old_project.id,
name=old_project.name,
path=old_project.path,
is_default=old_project.is_default or False,
name=name,
path=project_service.projects.get(name, ""),
)
if path:
@@ -100,21 +91,14 @@ async def update_project(
await project_service.update_project(name, is_active=is_active)
# Get updated project info
updated_project = await project_service.get_project(name)
if not updated_project:
raise HTTPException(status_code=404, detail=f"Project '{name}' not found after update")
updated_path = path if path else project_service.projects.get(name, "")
return ProjectStatusResponse(
message=f"Project '{name}' updated successfully",
status="success",
default=(name == project_service.default_project),
old_project=old_project_info,
new_project=ProjectItem(
id=updated_project.id,
name=updated_project.name,
path=updated_project.path,
is_default=updated_project.is_default or False,
),
new_project=ProjectItem(name=name, path=updated_path),
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
@@ -202,7 +186,6 @@ async def list_projects(
project_items = [
ProjectItem(
id=project.id,
name=project.name,
path=normalize_project_path(project.path),
is_default=project.is_default or False,
@@ -249,7 +232,6 @@ async def add_project(
status="success",
default=existing_project.is_default or False,
new_project=ProjectItem(
id=existing_project.id,
name=existing_project.name,
path=existing_project.path,
is_default=existing_project.is_default or False,
@@ -268,20 +250,12 @@ async def add_project(
project_data.name, project_data.path, set_default=project_data.set_default
)
# Fetch the newly created project to get its ID
new_project = await project_service.get_project(project_data.name)
if not new_project:
raise HTTPException(status_code=500, detail="Failed to retrieve newly created project")
return ProjectStatusResponse( # pyright: ignore [reportCallIssue]
message=f"Project '{project_data.name}' added successfully",
status="success",
default=project_data.set_default,
new_project=ProjectItem(
id=new_project.id,
name=new_project.name,
path=new_project.path,
is_default=new_project.is_default or False,
name=project_data.name, path=project_data.path, is_default=project_data.set_default
),
)
except ValueError as e: # pragma: no cover
@@ -332,12 +306,7 @@ async def remove_project(
message=f"Project '{name}' removed successfully",
status="success",
default=False,
old_project=ProjectItem(
id=old_project.id,
name=old_project.name,
path=old_project.path,
is_default=old_project.is_default or False,
),
old_project=ProjectItem(name=old_project.name, path=old_project.path),
new_project=None,
)
except ValueError as e: # pragma: no cover
@@ -380,14 +349,8 @@ async def set_default_project(
message=f"Project '{name}' set as default successfully",
status="success",
default=True,
old_project=ProjectItem(
id=default_project.id,
name=default_name,
path=default_project.path,
is_default=False,
),
old_project=ProjectItem(name=default_name, path=default_project.path),
new_project=ProjectItem(
id=new_default_project.id,
name=name,
path=new_default_project.path,
is_default=True,
@@ -415,12 +378,7 @@ async def get_default_project(
status_code=404, detail=f"Default Project: '{default_name}' does not exist"
)
return ProjectItem(
id=default_project.id,
name=default_project.name,
path=default_project.path,
is_default=True,
)
return ProjectItem(name=default_project.name, path=default_project.path, is_default=True)
# Synchronize projects between config and database
@@ -25,17 +25,6 @@ from datetime import datetime
router = APIRouter(prefix="/resource", tags=["resources"])
def _mtime_to_datetime(entity: EntityModel) -> datetime:
"""Convert entity mtime (file modification time) to datetime.
Returns the file's actual modification time, falling back to updated_at
if mtime is not available.
"""
if entity.mtime:
return datetime.fromtimestamp(entity.mtime).astimezone()
return entity.updated_at
def get_entity_ids(item: SearchIndexRow) -> set[int]:
match item.type:
case SearchItemType.ENTITY:
@@ -108,7 +97,7 @@ async def get_resource_content(
# Read content for each entity
content = await file_service.read_entity_content(result)
memory_url = normalize_memory_url(result.permalink)
modified_date = _mtime_to_datetime(result).isoformat()
modified_date = result.updated_at.isoformat()
checksum = result.checksum[:8] if result.checksum else ""
# Prepare the delimited content
-25
View File
@@ -29,7 +29,6 @@ async def to_graph_context(
match item.type:
case SearchItemType.ENTITY:
return EntitySummary(
entity_id=item.id,
title=item.title, # pyright: ignore
permalink=item.permalink,
content=item.content,
@@ -38,8 +37,6 @@ async def to_graph_context(
)
case SearchItemType.OBSERVATION:
return ObservationSummary(
observation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
title=item.title, # pyright: ignore
file_path=item.file_path,
category=item.category, # pyright: ignore
@@ -51,16 +48,12 @@ async def to_graph_context(
from_entity = await entity_repository.find_by_id(item.from_id) # pyright: ignore
to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
return RelationSummary(
relation_id=item.id,
entity_id=item.entity_id, # pyright: ignore
title=item.title, # pyright: ignore
file_path=item.file_path,
permalink=item.permalink, # pyright: ignore
relation_type=item.relation_type, # pyright: ignore
from_entity=from_entity.title if from_entity else None,
from_entity_id=item.from_id, # pyright: ignore
to_entity=to_entity.title if to_entity else None,
to_entity_id=item.to_id,
created_at=item.created_at,
)
case _: # pragma: no cover
@@ -118,21 +111,6 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
search_results = []
for r in results:
entities = await entity_service.get_entities_by_id([r.entity_id, r.from_id, r.to_id]) # pyright: ignore
# Determine which IDs to set based on type
entity_id = None
observation_id = None
relation_id = None
if r.type == SearchItemType.ENTITY:
entity_id = r.id
elif r.type == SearchItemType.OBSERVATION:
observation_id = r.id
entity_id = r.entity_id # Parent entity
elif r.type == SearchItemType.RELATION:
relation_id = r.id
entity_id = r.entity_id # Parent entity
search_results.append(
SearchResult(
title=r.title, # pyright: ignore
@@ -143,9 +121,6 @@ async def to_search_results(entity_service: EntityService, results: List[SearchI
content=r.content,
file_path=r.file_path,
metadata=r.metadata,
entity_id=entity_id,
observation_id=observation_id,
relation_id=relation_id,
category=r.category,
from_entity=entities[0].permalink if entities else None,
to_entity=entities[1].permalink if len(entities) > 1 else None,
-35
View File
@@ -1,35 +0,0 @@
"""API v2 module - ID-based entity references.
Version 2 of the Basic Memory API uses integer entity IDs as the primary
identifier for improved performance and stability.
Key changes from v1:
- Entity lookups use integer IDs instead of paths/permalinks
- Direct database queries instead of cascading resolution
- Stable references that don't change with file moves
- Better caching support
All v2 routers are registered with the /v2 prefix.
"""
from basic_memory.api.v2.routers import (
knowledge_router,
memory_router,
project_router,
resource_router,
search_router,
directory_router,
prompt_router,
importer_router,
)
__all__ = [
"knowledge_router",
"memory_router",
"project_router",
"resource_router",
"search_router",
"directory_router",
"prompt_router",
"importer_router",
]
@@ -1,21 +0,0 @@
"""V2 API routers."""
from basic_memory.api.v2.routers.knowledge_router import router as knowledge_router
from basic_memory.api.v2.routers.project_router import router as project_router
from basic_memory.api.v2.routers.memory_router import router as memory_router
from basic_memory.api.v2.routers.search_router import router as search_router
from basic_memory.api.v2.routers.resource_router import router as resource_router
from basic_memory.api.v2.routers.directory_router import router as directory_router
from basic_memory.api.v2.routers.prompt_router import router as prompt_router
from basic_memory.api.v2.routers.importer_router import router as importer_router
__all__ = [
"knowledge_router",
"project_router",
"memory_router",
"search_router",
"resource_router",
"directory_router",
"prompt_router",
"importer_router",
]
@@ -1,93 +0,0 @@
"""V2 Directory Router - ID-based directory tree operations.
This router provides directory structure browsing for projects using
integer project IDs instead of name-based identifiers.
Key improvements:
- Direct project lookup via integer primary keys
- Consistent with other v2 endpoints
- Better performance through indexed queries
"""
from typing import List, Optional
from fastapi import APIRouter, Query
from basic_memory.deps import DirectoryServiceV2Dep, ProjectIdPathDep
from basic_memory.schemas.directory import DirectoryNode
router = APIRouter(prefix="/directory", tags=["directory-v2"])
@router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True)
async def get_directory_tree(
directory_service: DirectoryServiceV2Dep,
project_id: ProjectIdPathDep,
):
"""Get hierarchical directory structure from the knowledge base.
Args:
directory_service: Service for directory operations
project_id: Numeric project ID
Returns:
DirectoryNode representing the root of the hierarchical tree structure
"""
# Get a hierarchical directory tree for the specific project
tree = await directory_service.get_directory_tree()
# Return the hierarchical tree
return tree
@router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True)
async def get_directory_structure(
directory_service: DirectoryServiceV2Dep,
project_id: ProjectIdPathDep,
):
"""Get folder structure for navigation (no files).
Optimized endpoint for folder tree navigation. Returns only directory nodes
without file metadata. For full tree with files, use /directory/tree.
Args:
directory_service: Service for directory operations
project_id: Numeric project ID
Returns:
DirectoryNode tree containing only folders (type="directory")
"""
structure = await directory_service.get_directory_structure()
return structure
@router.get("/list", response_model=List[DirectoryNode], response_model_exclude_none=True)
async def list_directory(
directory_service: DirectoryServiceV2Dep,
project_id: ProjectIdPathDep,
dir_name: str = Query("/", description="Directory path to list"),
depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"),
file_name_glob: Optional[str] = Query(
None, description="Glob pattern for filtering file names"
),
):
"""List directory contents with filtering and depth control.
Args:
directory_service: Service for directory operations
project_id: Numeric project ID
dir_name: Directory path to list (default: root "/")
depth: Recursion depth (1-10, default: 1 for immediate children only)
file_name_glob: Optional glob pattern for filtering file names (e.g., "*.md", "*meeting*")
Returns:
List of DirectoryNode objects matching the criteria
"""
# Get directory listing with filtering
nodes = await directory_service.list_directory(
dir_name=dir_name,
depth=depth,
file_name_glob=file_name_glob,
)
return nodes
@@ -1,182 +0,0 @@
"""V2 Import Router - ID-based data import operations.
This router uses v2 dependencies for consistent project ID handling.
Import endpoints use project_id in the path for consistency with other v2 endpoints.
"""
import json
import logging
from fastapi import APIRouter, Form, HTTPException, UploadFile, status
from basic_memory.deps import (
ChatGPTImporterV2Dep,
ClaudeConversationsImporterV2Dep,
ClaudeProjectsImporterV2Dep,
MemoryJsonImporterV2Dep,
ProjectIdPathDep,
)
from basic_memory.importers import Importer
from basic_memory.schemas.importer import (
ChatImportResult,
EntityImportResult,
ProjectImportResult,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/import", tags=["import-v2"])
@router.post("/chatgpt", response_model=ChatImportResult)
async def import_chatgpt(
project_id: ProjectIdPathDep,
importer: ChatGPTImporterV2Dep,
file: UploadFile,
folder: str = Form("conversations"),
) -> ChatImportResult:
"""Import conversations from ChatGPT JSON export.
Args:
project_id: Validated numeric project ID from URL path
file: The ChatGPT conversations.json file.
folder: The folder to place the files in.
importer: ChatGPT importer instance.
Returns:
ChatImportResult with import statistics.
Raises:
HTTPException: If import fails.
"""
logger.info(f"V2 Importing ChatGPT conversations for project {project_id}")
return await import_file(importer, file, folder)
@router.post("/claude/conversations", response_model=ChatImportResult)
async def import_claude_conversations(
project_id: ProjectIdPathDep,
importer: ClaudeConversationsImporterV2Dep,
file: UploadFile,
folder: str = Form("conversations"),
) -> ChatImportResult:
"""Import conversations from Claude conversations.json export.
Args:
project_id: Validated numeric project ID from URL path
file: The Claude conversations.json file.
folder: The folder to place the files in.
importer: Claude conversations importer instance.
Returns:
ChatImportResult with import statistics.
Raises:
HTTPException: If import fails.
"""
logger.info(f"V2 Importing Claude conversations for project {project_id}")
return await import_file(importer, file, folder)
@router.post("/claude/projects", response_model=ProjectImportResult)
async def import_claude_projects(
project_id: ProjectIdPathDep,
importer: ClaudeProjectsImporterV2Dep,
file: UploadFile,
folder: str = Form("projects"),
) -> ProjectImportResult:
"""Import projects from Claude projects.json export.
Args:
project_id: Validated numeric project ID from URL path
file: The Claude projects.json file.
folder: The base folder to place the files in.
importer: Claude projects importer instance.
Returns:
ProjectImportResult with import statistics.
Raises:
HTTPException: If import fails.
"""
logger.info(f"V2 Importing Claude projects for project {project_id}")
return await import_file(importer, file, folder)
@router.post("/memory-json", response_model=EntityImportResult)
async def import_memory_json(
project_id: ProjectIdPathDep,
importer: MemoryJsonImporterV2Dep,
file: UploadFile,
folder: str = Form("conversations"),
) -> EntityImportResult:
"""Import entities and relations from a memory.json file.
Args:
project_id: Validated numeric project ID from URL path
file: The memory.json file.
folder: Optional destination folder within the project.
importer: Memory JSON importer instance.
Returns:
EntityImportResult with import statistics.
Raises:
HTTPException: If import fails.
"""
logger.info(f"V2 Importing memory.json for project {project_id}")
try:
file_data = []
file_bytes = await file.read()
file_str = file_bytes.decode("utf-8")
for line in file_str.splitlines():
json_data = json.loads(line)
file_data.append(json_data)
result = await importer.import_data(file_data, folder)
if not result.success: # pragma: no cover
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=result.error_message or "Import failed",
)
except Exception as e:
logger.exception("V2 Import failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Import failed: {str(e)}",
)
return result
async def import_file(importer: Importer, file: UploadFile, destination_folder: str):
"""Helper function to import a file using an importer instance.
Args:
importer: The importer instance to use
file: The file to import
destination_folder: Destination folder for imported content
Returns:
Import result from the importer
Raises:
HTTPException: If import fails
"""
try:
# Process file
json_data = json.load(file.file)
result = await importer.import_data(json_data, destination_folder)
if not result.success: # pragma: no cover
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=result.error_message or "Import failed",
)
return result
except Exception as e:
logger.exception("V2 Import failed")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Import failed: {str(e)}",
)
@@ -1,415 +0,0 @@
"""V2 Knowledge Router - ID-based entity operations.
This router provides ID-based CRUD operations for entities, replacing the
path-based identifiers used in v1 with direct integer ID lookups.
Key improvements:
- Direct database lookups via integer primary keys
- Stable references that don't change with file moves
- Better performance through indexed queries
- Simplified caching strategies
"""
from fastapi import APIRouter, HTTPException, BackgroundTasks, Depends, Response
from loguru import logger
from basic_memory.deps import (
EntityServiceV2Dep,
SearchServiceV2Dep,
LinkResolverV2Dep,
ProjectConfigV2Dep,
AppConfigDep,
SyncServiceV2Dep,
EntityRepositoryV2Dep,
ProjectIdPathDep,
)
from basic_memory.schemas import DeleteEntitiesResponse
from basic_memory.schemas.base import Entity
from basic_memory.schemas.request import EditEntityRequest
from basic_memory.schemas.v2 import (
EntityResolveRequest,
EntityResolveResponse,
EntityResponseV2,
MoveEntityRequestV2,
)
router = APIRouter(prefix="/knowledge", tags=["knowledge-v2"])
async def resolve_relations_background(sync_service, entity_id: int, entity_permalink: str) -> None:
"""Background task to resolve relations for a specific entity.
This runs asynchronously after the API response is sent, preventing
long delays when creating entities with many relations.
"""
try:
# Only resolve relations for the newly created entity
await sync_service.resolve_relations(entity_id=entity_id)
logger.debug(
f"Background: Resolved relations for entity {entity_permalink} (id={entity_id})"
)
except Exception as e:
# Log but don't fail - this is a background task
logger.warning(
f"Background: Failed to resolve relations for entity {entity_permalink}: {e}"
)
## Resolution endpoint
@router.post("/resolve", response_model=EntityResolveResponse)
async def resolve_identifier(
project_id: ProjectIdPathDep,
data: EntityResolveRequest,
link_resolver: LinkResolverV2Dep,
) -> EntityResolveResponse:
"""Resolve a string identifier (permalink, title, or path) to an entity ID.
This endpoint provides a bridge between v1-style identifiers and v2 entity IDs.
Use this to convert existing references to the new ID-based format.
Args:
data: Request containing the identifier to resolve
Returns:
Entity ID and metadata about how it was resolved
Raises:
HTTPException: 404 if identifier cannot be resolved
Example:
POST /v2/{project}/knowledge/resolve
{"identifier": "specs/search"}
Returns:
{
"entity_id": 123,
"permalink": "specs/search",
"file_path": "specs/search.md",
"title": "Search Specification",
"resolution_method": "permalink"
}
"""
logger.info(f"API v2 request: resolve_identifier for '{data.identifier}'")
# Try to resolve the identifier
entity = await link_resolver.resolve_link(data.identifier)
if not entity:
raise HTTPException(
status_code=404, detail=f"Could not resolve identifier: '{data.identifier}'"
)
# Determine resolution method
resolution_method = "search" # default
if data.identifier.isdigit():
resolution_method = "id"
elif entity.permalink == data.identifier:
resolution_method = "permalink"
elif entity.title == data.identifier:
resolution_method = "title"
elif entity.file_path == data.identifier:
resolution_method = "path"
result = EntityResolveResponse(
entity_id=entity.id,
permalink=entity.permalink,
file_path=entity.file_path,
title=entity.title,
resolution_method=resolution_method,
)
logger.info(
f"API v2 response: resolved '{data.identifier}' to entity_id={result.entity_id} via {resolution_method}"
)
return result
## Read endpoints
@router.get("/entities/{entity_id}", response_model=EntityResponseV2)
async def get_entity_by_id(
project_id: ProjectIdPathDep,
entity_id: int,
entity_repository: EntityRepositoryV2Dep,
) -> EntityResponseV2:
"""Get an entity by its numeric ID.
This is the primary entity retrieval method in v2, using direct database
lookups for maximum performance.
Args:
entity_id: Numeric entity ID
Returns:
Complete entity with observations and relations
Raises:
HTTPException: 404 if entity not found
"""
logger.info(f"API v2 request: get_entity_by_id entity_id={entity_id}")
entity = await entity_repository.get_by_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
result = EntityResponseV2.model_validate(entity)
logger.info(f"API v2 response: entity_id={entity_id}, title='{result.title}'")
return result
## Create endpoints
@router.post("/entities", response_model=EntityResponseV2)
async def create_entity(
project_id: ProjectIdPathDep,
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2Dep,
search_service: SearchServiceV2Dep,
) -> EntityResponseV2:
"""Create a new entity.
Args:
data: Entity data to create
Returns:
Created entity with generated ID
"""
logger.info(
"API v2 request", endpoint="create_entity", entity_type=data.entity_type, title=data.title
)
entity = await entity_service.create_entity(data)
# reindex
await search_service.index_entity(entity, background_tasks=background_tasks)
result = EntityResponseV2.model_validate(entity)
logger.info(
f"API v2 response: endpoint='create_entity' id={entity.id}, title={result.title}, permalink={result.permalink}, status_code=201"
)
return result
## Update endpoints
@router.put("/entities/{entity_id}", response_model=EntityResponseV2)
async def update_entity_by_id(
project_id: ProjectIdPathDep,
entity_id: int,
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2Dep,
search_service: SearchServiceV2Dep,
sync_service: SyncServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
) -> EntityResponseV2:
"""Update an entity by ID.
If the entity doesn't exist, it will be created (upsert behavior).
Args:
entity_id: Numeric entity ID
data: Updated entity data
Returns:
Updated entity
"""
logger.info(f"API v2 request: update_entity_by_id entity_id={entity_id}")
# Check if entity exists
existing = await entity_repository.get_by_id(entity_id)
created = existing is None
# Perform update or create
entity, _ = await entity_service.create_or_update_entity(data)
response.status_code = 201 if created else 200
# reindex
await search_service.index_entity(entity, background_tasks=background_tasks)
# Schedule relation resolution for new entities
if created:
background_tasks.add_task(
resolve_relations_background, sync_service, entity.id, entity.permalink or ""
)
result = EntityResponseV2.model_validate(entity)
logger.info(
f"API v2 response: entity_id={entity_id}, created={created}, status_code={response.status_code}"
)
return result
@router.patch("/entities/{entity_id}", response_model=EntityResponseV2)
async def edit_entity_by_id(
project_id: ProjectIdPathDep,
entity_id: int,
data: EditEntityRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2Dep,
search_service: SearchServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
) -> EntityResponseV2:
"""Edit an existing entity by ID using operations like append, prepend, etc.
Args:
entity_id: Numeric entity ID
data: Edit operation details
Returns:
Updated entity
Raises:
HTTPException: 404 if entity not found, 400 if edit fails
"""
logger.info(
f"API v2 request: edit_entity_by_id entity_id={entity_id}, operation='{data.operation}'"
)
# Verify entity exists
entity = await entity_repository.get_by_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
try:
# Edit using the entity's permalink or path
identifier = entity.permalink or entity.file_path
updated_entity = await entity_service.edit_entity(
identifier=identifier,
operation=data.operation,
content=data.content,
section=data.section,
find_text=data.find_text,
expected_replacements=data.expected_replacements,
)
# Reindex
await search_service.index_entity(updated_entity, background_tasks=background_tasks)
result = EntityResponseV2.model_validate(updated_entity)
logger.info(
f"API v2 response: entity_id={entity_id}, operation='{data.operation}', status_code=200"
)
return result
except Exception as e:
logger.error(f"Error editing entity {entity_id}: {e}")
raise HTTPException(status_code=400, detail=str(e))
## Delete endpoints
@router.delete("/entities/{entity_id}", response_model=DeleteEntitiesResponse)
async def delete_entity_by_id(
project_id: ProjectIdPathDep,
entity_id: int,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
search_service=Depends(lambda: None), # Optional for now
) -> DeleteEntitiesResponse:
"""Delete an entity by ID.
Args:
entity_id: Numeric entity ID
Returns:
Deletion status
Note: Returns deleted=False if entity doesn't exist (idempotent)
"""
logger.info(f"API v2 request: delete_entity_by_id entity_id={entity_id}")
entity = await entity_repository.get_by_id(entity_id)
if entity is None:
logger.info(f"API v2 response: entity_id={entity_id} not found, deleted=False")
return DeleteEntitiesResponse(deleted=False)
# Delete the entity
deleted = await entity_service.delete_entity(entity_id)
# Remove from search index if search service available
if search_service:
background_tasks.add_task(search_service.handle_delete, entity)
logger.info(f"API v2 response: entity_id={entity_id}, deleted={deleted}")
return DeleteEntitiesResponse(deleted=deleted)
## Move endpoint
@router.put("/entities/{entity_id}/move", response_model=EntityResponseV2)
async def move_entity(
project_id: ProjectIdPathDep,
entity_id: int,
data: MoveEntityRequestV2,
background_tasks: BackgroundTasks,
entity_service: EntityServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
project_config: ProjectConfigV2Dep,
app_config: AppConfigDep,
search_service: SearchServiceV2Dep,
) -> EntityResponseV2:
"""Move an entity to a new file location.
V2 API uses entity ID in the URL path for stable references.
The entity ID will remain stable after the move.
Args:
project_id: Project ID from URL path
entity_id: Entity ID from URL path (primary identifier)
data: Move request with destination path only
Returns:
Updated entity with new file path
"""
logger.info(
f"API v2 request: move_entity entity_id={entity_id}, destination='{data.destination_path}'"
)
try:
# First, get the entity by ID to verify it exists
entity = await entity_repository.find_by_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity not found: {entity_id}")
# Move the entity using its current file path as identifier
moved_entity = await entity_service.move_entity(
identifier=entity.file_path, # Use file path for resolution
destination_path=data.destination_path,
project_config=project_config,
app_config=app_config,
)
# Reindex at new location
reindexed_entity = await entity_service.link_resolver.resolve_link(data.destination_path)
if reindexed_entity:
await search_service.index_entity(reindexed_entity, background_tasks=background_tasks)
result = EntityResponseV2.model_validate(moved_entity)
logger.info(
f"API v2 response: moved entity_id={moved_entity.id} to '{data.destination_path}'"
)
return result
except HTTPException:
raise
except Exception as e:
logger.error(f"Error moving entity: {e}")
raise HTTPException(status_code=400, detail=str(e))
@@ -1,130 +0,0 @@
"""V2 routes for memory:// URI operations.
This router uses integer project IDs for stable, efficient routing.
V1 uses string-based project names which are less efficient and less stable.
"""
from typing import Annotated, Optional
from fastapi import APIRouter, Query
from loguru import logger
from basic_memory.deps import ContextServiceV2Dep, EntityRepositoryV2Dep, ProjectIdPathDep
from basic_memory.schemas.base import TimeFrame, parse_timeframe
from basic_memory.schemas.memory import (
GraphContext,
normalize_memory_url,
)
from basic_memory.schemas.search import SearchItemType
from basic_memory.api.routers.utils import to_graph_context
# Note: No prefix here - it's added during registration as /v2/{project_id}/memory
router = APIRouter(tags=["memory"])
@router.get("/memory/recent", response_model=GraphContext)
async def recent(
project_id: ProjectIdPathDep,
context_service: ContextServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
type: Annotated[list[SearchItemType] | None, Query()] = None,
depth: int = 1,
timeframe: TimeFrame = "7d",
page: int = 1,
page_size: int = 10,
max_related: int = 10,
) -> GraphContext:
"""Get recent activity context for a project.
Args:
project_id: Validated numeric project ID from URL path
context_service: Context service scoped to project
entity_repository: Entity repository scoped to project
type: Types of items to include (entities, relations, observations)
depth: How many levels of related entities to include
timeframe: Time window for recent activity (e.g., "7d", "1 week")
page: Page number for pagination
page_size: Number of items per page
max_related: Maximum related entities to include per item
Returns:
GraphContext with recent activity and related entities
"""
# return all types by default
types = (
[SearchItemType.ENTITY, SearchItemType.RELATION, SearchItemType.OBSERVATION]
if not type
else type
)
logger.debug(
f"V2 Getting recent context for project {project_id}: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
# Parse timeframe
since = parse_timeframe(timeframe)
limit = page_size
offset = (page - 1) * page_size
# Build context
context = await context_service.build_context(
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
)
recent_context = await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
logger.debug(f"V2 Recent context: {recent_context.model_dump_json()}")
return recent_context
# get_memory_context needs to be declared last so other paths can match
@router.get("/memory/{uri:path}", response_model=GraphContext)
async def get_memory_context(
project_id: ProjectIdPathDep,
context_service: ContextServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
uri: str,
depth: int = 1,
timeframe: Optional[TimeFrame] = None,
page: int = 1,
page_size: int = 10,
max_related: int = 10,
) -> GraphContext:
"""Get rich context from memory:// URI.
V2 supports both legacy path-based URIs and new ID-based URIs:
- Legacy: memory://path/to/note
- ID-based: memory://id/123 or memory://123
Args:
project_id: Validated numeric project ID from URL path
context_service: Context service scoped to project
entity_repository: Entity repository scoped to project
uri: Memory URI path (e.g., "id/123", "123", or "path/to/note")
depth: How many levels of related entities to include
timeframe: Optional time window for filtering related content
page: Page number for pagination
page_size: Number of items per page
max_related: Maximum related entities to include
Returns:
GraphContext with the entity and its related context
"""
logger.debug(
f"V2 Getting context for project {project_id}, URI: `{uri}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
memory_url = normalize_memory_url(uri)
# Parse timeframe
since = parse_timeframe(timeframe) if timeframe else None
limit = page_size
offset = (page - 1) * page_size
# Build context
context = await context_service.build_context(
memory_url, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
)
return await to_graph_context(
context, entity_repository=entity_repository, page=page, page_size=page_size
)
@@ -1,264 +0,0 @@
"""V2 Project Router - ID-based project management operations.
This router provides ID-based CRUD operations for projects, replacing the
name-based identifiers used in v1 with direct integer ID lookups.
Key improvements:
- Direct database lookups via integer primary keys
- Stable references that don't change with project renames
- Better performance through indexed queries
- Consistent with v2 entity operations
"""
import os
from typing import Optional
from fastapi import APIRouter, HTTPException, Body, Query
from loguru import logger
from basic_memory.deps import (
ProjectServiceDep,
ProjectRepositoryDep,
ProjectIdPathDep,
)
from basic_memory.schemas.project_info import (
ProjectItem,
ProjectStatusResponse,
)
from basic_memory.utils import normalize_project_path
router = APIRouter(prefix="/projects", tags=["project_management-v2"])
@router.get("/{project_id}", response_model=ProjectItem)
async def get_project_by_id(
project_id: ProjectIdPathDep,
project_repository: ProjectRepositoryDep,
) -> ProjectItem:
"""Get project by its numeric ID.
This is the primary project retrieval method in v2, using direct database
lookups for maximum performance.
Args:
project_id: Numeric project ID
Returns:
Project information
Raises:
HTTPException: 404 if project not found
Example:
GET /v2/projects/3
"""
logger.info(f"API v2 request: get_project_by_id for project_id={project_id}")
project = await project_repository.get_by_id(project_id)
if not project:
raise HTTPException(status_code=404, detail=f"Project with ID {project_id} not found")
return ProjectItem(
id=project.id,
name=project.name,
path=normalize_project_path(project.path),
is_default=project.is_default or False,
)
@router.patch("/{project_id}", response_model=ProjectStatusResponse)
async def update_project_by_id(
project_id: ProjectIdPathDep,
project_service: ProjectServiceDep,
project_repository: ProjectRepositoryDep,
path: Optional[str] = Body(None, description="New absolute path for the project"),
is_active: Optional[bool] = Body(None, description="Status of the project (active/inactive)"),
) -> ProjectStatusResponse:
"""Update a project's information by ID.
Args:
project_id: Numeric project ID
path: Optional new absolute path for the project
is_active: Optional status update for the project
Returns:
Response confirming the project was updated
Raises:
HTTPException: 400 if validation fails, 404 if project not found
Example:
PATCH /v2/projects/3
{"path": "/new/path"}
"""
logger.info(f"API v2 request: update_project_by_id for project_id={project_id}")
try:
# Validate that path is absolute if provided
if path and not os.path.isabs(path):
raise HTTPException(status_code=400, detail="Path must be absolute")
# Get original project info for the response
old_project = await project_repository.get_by_id(project_id)
if not old_project:
raise HTTPException(status_code=404, detail=f"Project with ID {project_id} not found")
old_project_info = ProjectItem(
id=old_project.id,
name=old_project.name,
path=old_project.path,
is_default=old_project.is_default or False,
)
# Update using project name (service layer still uses names internally)
if path:
await project_service.move_project(old_project.name, path)
elif is_active is not None:
await project_service.update_project(old_project.name, is_active=is_active)
# Get updated project info
updated_project = await project_repository.get_by_id(project_id)
if not updated_project:
raise HTTPException(
status_code=404, detail=f"Project with ID {project_id} not found after update"
)
return ProjectStatusResponse(
message=f"Project '{updated_project.name}' updated successfully",
status="success",
default=(old_project.name == project_service.default_project),
old_project=old_project_info,
new_project=ProjectItem(
id=updated_project.id,
name=updated_project.name,
path=updated_project.path,
is_default=updated_project.is_default or False,
),
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
@router.delete("/{project_id}", response_model=ProjectStatusResponse)
async def delete_project_by_id(
project_id: ProjectIdPathDep,
project_service: ProjectServiceDep,
project_repository: ProjectRepositoryDep,
delete_notes: bool = Query(
False, description="If True, delete project directory from filesystem"
),
) -> ProjectStatusResponse:
"""Delete a project by ID.
Args:
project_id: Numeric project ID
delete_notes: If True, delete the project directory from the filesystem
Returns:
Response confirming the project was deleted
Raises:
HTTPException: 400 if trying to delete default project, 404 if not found
Example:
DELETE /v2/projects/3?delete_notes=false
"""
logger.info(
f"API v2 request: delete_project_by_id for project_id={project_id}, delete_notes={delete_notes}"
)
try:
old_project = await project_repository.get_by_id(project_id)
if not old_project:
raise HTTPException(status_code=404, detail=f"Project with ID {project_id} not found")
# Check if trying to delete the default project
if old_project.name == project_service.default_project:
available_projects = await project_service.list_projects()
other_projects = [p.name for p in available_projects if p.id != project_id]
detail = f"Cannot delete default project '{old_project.name}'. "
if other_projects:
detail += (
f"Set another project as default first. Available: {', '.join(other_projects)}"
)
else:
detail += "This is the only project in your configuration."
raise HTTPException(status_code=400, detail=detail)
# Delete using project name (service layer still uses names internally)
await project_service.remove_project(old_project.name, delete_notes=delete_notes)
return ProjectStatusResponse(
message=f"Project '{old_project.name}' removed successfully",
status="success",
default=False,
old_project=ProjectItem(
id=old_project.id,
name=old_project.name,
path=old_project.path,
is_default=old_project.is_default or False,
),
new_project=None,
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
@router.put("/{project_id}/default", response_model=ProjectStatusResponse)
async def set_default_project_by_id(
project_id: ProjectIdPathDep,
project_service: ProjectServiceDep,
project_repository: ProjectRepositoryDep,
) -> ProjectStatusResponse:
"""Set a project as the default project by ID.
Args:
project_id: Numeric project ID to set as default
Returns:
Response confirming the project was set as default
Raises:
HTTPException: 404 if project not found
Example:
PUT /v2/projects/3/default
"""
logger.info(f"API v2 request: set_default_project_by_id for project_id={project_id}")
try:
# Get the old default project
default_name = project_service.default_project
default_project = await project_service.get_project(default_name)
if not default_project:
raise HTTPException(
status_code=404, detail=f"Default Project: '{default_name}' does not exist"
)
# Get the new default project
new_default_project = await project_repository.get_by_id(project_id)
if not new_default_project:
raise HTTPException(status_code=404, detail=f"Project with ID {project_id} not found")
# Set as default using project name (service layer still uses names internally)
await project_service.set_default_project(new_default_project.name)
return ProjectStatusResponse(
message=f"Project '{new_default_project.name}' set as default successfully",
status="success",
default=True,
old_project=ProjectItem(
id=default_project.id,
name=default_name,
path=default_project.path,
is_default=False,
),
new_project=ProjectItem(
id=new_default_project.id,
name=new_default_project.name,
path=new_default_project.path,
is_default=True,
),
)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
@@ -1,270 +0,0 @@
"""V2 Prompt Router - ID-based prompt generation operations.
This router uses v2 dependencies for consistent project ID handling.
Prompt endpoints are action-based (not resource-based), so they don't
have entity IDs in URLs - they generate formatted prompts from queries.
"""
from datetime import datetime, timezone
from fastapi import APIRouter, HTTPException, status
from loguru import logger
from basic_memory.api.routers.utils import to_graph_context, to_search_results
from basic_memory.api.template_loader import template_loader
from basic_memory.schemas.base import parse_timeframe
from basic_memory.deps import (
ContextServiceV2Dep,
EntityRepositoryV2Dep,
SearchServiceV2Dep,
EntityServiceV2Dep,
ProjectIdPathDep,
)
from basic_memory.schemas.prompt import (
ContinueConversationRequest,
SearchPromptRequest,
PromptResponse,
PromptMetadata,
)
from basic_memory.schemas.search import SearchItemType, SearchQuery
router = APIRouter(prefix="/prompt", tags=["prompt-v2"])
@router.post("/continue-conversation", response_model=PromptResponse)
async def continue_conversation(
project_id: ProjectIdPathDep,
search_service: SearchServiceV2Dep,
entity_service: EntityServiceV2Dep,
context_service: ContextServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
request: ContinueConversationRequest,
) -> PromptResponse:
"""Generate a prompt for continuing a conversation.
This endpoint takes a topic and/or timeframe and generates a prompt with
relevant context from the knowledge base.
Args:
project_id: Validated numeric project ID from URL path
request: The request parameters
Returns:
Formatted continuation prompt with context
"""
logger.info(
f"V2 Generating continue conversation prompt for project {project_id}, "
f"topic: {request.topic}, timeframe: {request.timeframe}"
)
since = parse_timeframe(request.timeframe) if request.timeframe else None
# Initialize search results
search_results = []
# Get data needed for template
if request.topic:
query = SearchQuery(text=request.topic, after_date=request.timeframe)
results = await search_service.search(query, limit=request.search_items_limit)
search_results = await to_search_results(entity_service, results)
# Build context from results
all_hierarchical_results = []
for result in search_results:
if hasattr(result, "permalink") and result.permalink:
# Get hierarchical context using the new dataclass-based approach
context_result = await context_service.build_context(
result.permalink,
depth=request.depth,
since=since,
max_related=request.related_items_limit,
include_observations=True, # Include observations for entities
)
# Process results into the schema format
graph_context = await to_graph_context(
context_result, entity_repository=entity_repository
)
# Add results to our collection (limit to top results for each permalink)
if graph_context.results:
all_hierarchical_results.extend(graph_context.results[:3])
# Limit to a reasonable number of total results
all_hierarchical_results = all_hierarchical_results[:10]
template_context = {
"topic": request.topic,
"timeframe": request.timeframe,
"hierarchical_results": all_hierarchical_results,
"has_results": len(all_hierarchical_results) > 0,
}
else:
# If no topic, get recent activity
context_result = await context_service.build_context(
types=[SearchItemType.ENTITY],
depth=request.depth,
since=since,
max_related=request.related_items_limit,
include_observations=True,
)
recent_context = await to_graph_context(context_result, entity_repository=entity_repository)
hierarchical_results = recent_context.results[:5] # Limit to top 5 recent items
template_context = {
"topic": f"Recent Activity from ({request.timeframe})",
"timeframe": request.timeframe,
"hierarchical_results": hierarchical_results,
"has_results": len(hierarchical_results) > 0,
}
try:
# Render template
rendered_prompt = await template_loader.render(
"prompts/continue_conversation.hbs", template_context
)
# Calculate metadata
# Count items of different types
observation_count = 0
relation_count = 0
entity_count = 0
# Get the hierarchical results from the template context
hierarchical_results_for_count = template_context.get("hierarchical_results", [])
# For topic-based search
if request.topic:
for item in hierarchical_results_for_count:
if hasattr(item, "observations"):
observation_count += len(item.observations) if item.observations else 0
if hasattr(item, "related_results"):
for related in item.related_results or []:
if hasattr(related, "type"):
if related.type == "relation":
relation_count += 1
elif related.type == "entity": # pragma: no cover
entity_count += 1 # pragma: no cover
# For recent activity
else:
for item in hierarchical_results_for_count:
if hasattr(item, "observations"):
observation_count += len(item.observations) if item.observations else 0
if hasattr(item, "related_results"):
for related in item.related_results or []:
if hasattr(related, "type"):
if related.type == "relation":
relation_count += 1
elif related.type == "entity": # pragma: no cover
entity_count += 1 # pragma: no cover
# Build metadata
metadata = {
"query": request.topic,
"timeframe": request.timeframe,
"search_count": len(search_results)
if request.topic
else 0, # Original search results count
"context_count": len(hierarchical_results_for_count),
"observation_count": observation_count,
"relation_count": relation_count,
"total_items": (
len(hierarchical_results_for_count)
+ observation_count
+ relation_count
+ entity_count
),
"search_limit": request.search_items_limit,
"context_depth": request.depth,
"related_limit": request.related_items_limit,
"generated_at": datetime.now(timezone.utc).isoformat(),
}
prompt_metadata = PromptMetadata(**metadata)
return PromptResponse(
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
)
except Exception as e:
logger.error(f"Error rendering continue conversation template: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error rendering prompt template: {str(e)}",
)
@router.post("/search", response_model=PromptResponse)
async def search_prompt(
project_id: ProjectIdPathDep,
search_service: SearchServiceV2Dep,
entity_service: EntityServiceV2Dep,
request: SearchPromptRequest,
page: int = 1,
page_size: int = 10,
) -> PromptResponse:
"""Generate a prompt for search results.
This endpoint takes a search query and formats the results into a helpful
prompt with context and suggestions.
Args:
project_id: Validated numeric project ID from URL path
request: The search parameters
page: The page number for pagination
page_size: The number of results per page, defaults to 10
Returns:
Formatted search results prompt with context
"""
logger.info(
f"V2 Generating search prompt for project {project_id}, "
f"query: {request.query}, timeframe: {request.timeframe}"
)
limit = page_size
offset = (page - 1) * page_size
query = SearchQuery(text=request.query, after_date=request.timeframe)
results = await search_service.search(query, limit=limit, offset=offset)
search_results = await to_search_results(entity_service, results)
template_context = {
"query": request.query,
"timeframe": request.timeframe,
"results": search_results,
"has_results": len(search_results) > 0,
"result_count": len(search_results),
}
try:
# Render template
rendered_prompt = await template_loader.render("prompts/search.hbs", template_context)
# Build metadata
metadata = {
"query": request.query,
"timeframe": request.timeframe,
"search_count": len(search_results),
"context_count": len(search_results),
"observation_count": 0, # Search results don't include observations
"relation_count": 0, # Search results don't include relations
"total_items": len(search_results),
"search_limit": limit,
"context_depth": 0, # No context depth for basic search
"related_limit": 0, # No related items for basic search
"generated_at": datetime.now(timezone.utc).isoformat(),
}
prompt_metadata = PromptMetadata(**metadata)
return PromptResponse(
prompt=rendered_prompt, context=template_context, metadata=prompt_metadata
)
except Exception as e:
logger.error(f"Error rendering search template: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Error rendering prompt template: {str(e)}",
)
@@ -1,292 +0,0 @@
"""V2 Resource Router - ID-based resource content operations.
This router uses entity IDs for all operations, with file paths in request bodies
when needed. This is consistent with v2's ID-first design.
Key differences from v1:
- Uses integer entity IDs in URL paths instead of file paths
- File paths are in request bodies for create/update operations
- More RESTful: POST for create, PUT for update, GET for read
"""
from pathlib import Path
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
from loguru import logger
from basic_memory.deps import (
ProjectConfigV2Dep,
EntityServiceV2Dep,
FileServiceV2Dep,
EntityRepositoryV2Dep,
SearchServiceV2Dep,
ProjectIdPathDep,
)
from basic_memory.models.knowledge import Entity as EntityModel
from basic_memory.schemas.v2.resource import (
CreateResourceRequest,
UpdateResourceRequest,
ResourceResponse,
)
from basic_memory.utils import validate_project_path
from datetime import datetime
router = APIRouter(prefix="/resource", tags=["resources-v2"])
@router.get("/{entity_id}")
async def get_resource_content(
project_id: ProjectIdPathDep,
entity_id: int,
config: ProjectConfigV2Dep,
entity_service: EntityServiceV2Dep,
file_service: FileServiceV2Dep,
) -> FileResponse:
"""Get raw resource content by entity ID.
Args:
project_id: Validated numeric project ID from URL path
entity_id: Numeric entity ID
config: Project configuration
entity_service: Entity service for fetching entity data
file_service: File service for reading file content
Returns:
FileResponse with entity content
Raises:
HTTPException: 404 if entity or file not found
"""
logger.debug(f"V2 Getting content for project {project_id}, entity_id: {entity_id}")
# Get entity by ID
entities = await entity_service.get_entities_by_id([entity_id])
if not entities:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
entity = entities[0]
# Validate entity file path to prevent path traversal
project_path = Path(config.home)
if not validate_project_path(entity.file_path, project_path):
logger.error(f"Invalid file path in entity {entity.id}: {entity.file_path}")
raise HTTPException(
status_code=500,
detail="Entity contains invalid file path",
)
file_path = Path(f"{config.home}/{entity.file_path}")
if not file_path.exists():
raise HTTPException(
status_code=404,
detail=f"File not found: {file_path}",
)
return FileResponse(path=file_path)
@router.post("", response_model=ResourceResponse)
async def create_resource(
project_id: ProjectIdPathDep,
data: CreateResourceRequest,
config: ProjectConfigV2Dep,
file_service: FileServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
search_service: SearchServiceV2Dep,
) -> ResourceResponse:
"""Create a new resource file.
Args:
project_id: Validated numeric project ID from URL path
data: Create resource request with file_path and content
config: Project configuration
file_service: File service for writing files
entity_repository: Entity repository for creating entities
search_service: Search service for indexing
Returns:
ResourceResponse with file information including entity_id
Raises:
HTTPException: 400 for invalid file paths, 409 if file already exists
"""
try:
# Validate path to prevent path traversal attacks
project_path = Path(config.home)
if not validate_project_path(data.file_path, project_path):
logger.warning(
f"Invalid file path attempted: {data.file_path} in project {config.name}"
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {data.file_path}. "
"Path must be relative and stay within project boundaries.",
)
# Check if entity already exists
existing_entity = await entity_repository.get_by_file_path(data.file_path)
if existing_entity:
raise HTTPException(
status_code=409,
detail=f"Resource already exists at {data.file_path} with entity_id {existing_entity.id}. "
f"Use PUT /resource/{existing_entity.id} to update it.",
)
# Get full file path
full_path = Path(f"{config.home}/{data.file_path}")
# Ensure parent directory exists
full_path.parent.mkdir(parents=True, exist_ok=True)
# Write content to file
checksum = await file_service.write_file(full_path, data.content)
# Get file info
file_stats = file_service.file_stats(full_path)
# Determine file details
file_name = Path(data.file_path).name
content_type = file_service.content_type(full_path)
entity_type = "canvas" if data.file_path.endswith(".canvas") else "file"
# Create a new entity model
entity = EntityModel(
title=file_name,
entity_type=entity_type,
content_type=content_type,
file_path=data.file_path,
checksum=checksum,
created_at=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
updated_at=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
)
entity = await entity_repository.add(entity)
# Index the file for search
await search_service.index_entity(entity) # pyright: ignore
# Return success response
return ResourceResponse(
entity_id=entity.id,
file_path=data.file_path,
checksum=checksum,
size=file_stats.st_size,
created_at=file_stats.st_ctime,
modified_at=file_stats.st_mtime,
)
except HTTPException:
# Re-raise HTTP exceptions without wrapping
raise
except Exception as e: # pragma: no cover
logger.error(f"Error creating resource {data.file_path}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to create resource: {str(e)}")
@router.put("/{entity_id}", response_model=ResourceResponse)
async def update_resource(
project_id: ProjectIdPathDep,
entity_id: int,
data: UpdateResourceRequest,
config: ProjectConfigV2Dep,
file_service: FileServiceV2Dep,
entity_repository: EntityRepositoryV2Dep,
search_service: SearchServiceV2Dep,
) -> ResourceResponse:
"""Update an existing resource by entity ID.
Can update content and optionally move the file to a new path.
Args:
project_id: Validated numeric project ID from URL path
entity_id: Entity ID of the resource to update
data: Update resource request with content and optional new file_path
config: Project configuration
file_service: File service for writing files
entity_repository: Entity repository for updating entities
search_service: Search service for indexing
Returns:
ResourceResponse with updated file information
Raises:
HTTPException: 404 if entity not found, 400 for invalid paths
"""
try:
# Get existing entity
entity = await entity_repository.get_by_id(entity_id)
if not entity:
raise HTTPException(status_code=404, detail=f"Entity {entity_id} not found")
# Determine target file path
target_file_path = data.file_path if data.file_path else entity.file_path
# Validate path to prevent path traversal attacks
project_path = Path(config.home)
if not validate_project_path(target_file_path, project_path):
logger.warning(
f"Invalid file path attempted: {target_file_path} in project {config.name}"
)
raise HTTPException(
status_code=400,
detail=f"Invalid file path: {target_file_path}. "
"Path must be relative and stay within project boundaries.",
)
# Get full paths
old_full_path = Path(f"{config.home}/{entity.file_path}")
new_full_path = Path(f"{config.home}/{target_file_path}")
# If moving file, handle the move
if data.file_path and data.file_path != entity.file_path:
# Ensure new parent directory exists
new_full_path.parent.mkdir(parents=True, exist_ok=True)
# If old file exists, remove it
if old_full_path.exists():
old_full_path.unlink()
else:
# Ensure directory exists for in-place update
new_full_path.parent.mkdir(parents=True, exist_ok=True)
# Write content to target file
checksum = await file_service.write_file(new_full_path, data.content)
# Get file info
file_stats = file_service.file_stats(new_full_path)
# Determine file details
file_name = Path(target_file_path).name
content_type = file_service.content_type(new_full_path)
entity_type = "canvas" if target_file_path.endswith(".canvas") else "file"
# Update entity
updated_entity = await entity_repository.update(
entity_id,
{
"title": file_name,
"entity_type": entity_type,
"content_type": content_type,
"file_path": target_file_path,
"checksum": checksum,
"updated_at": datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
},
)
# Index the updated file for search
await search_service.index_entity(updated_entity) # pyright: ignore
# Return success response
return ResourceResponse(
entity_id=entity_id,
file_path=target_file_path,
checksum=checksum,
size=file_stats.st_size,
created_at=file_stats.st_ctime,
modified_at=file_stats.st_mtime,
)
except HTTPException:
# Re-raise HTTP exceptions without wrapping
raise
except Exception as e: # pragma: no cover
logger.error(f"Error updating resource {entity_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to update resource: {str(e)}")
@@ -1,73 +0,0 @@
"""V2 router for search operations.
This router uses integer project IDs for stable, efficient routing.
V1 uses string-based project names which are less efficient and less stable.
"""
from fastapi import APIRouter, BackgroundTasks
from basic_memory.api.routers.utils import to_search_results
from basic_memory.schemas.search import SearchQuery, SearchResponse
from basic_memory.deps import SearchServiceV2Dep, EntityServiceV2Dep, ProjectIdPathDep
# Note: No prefix here - it's added during registration as /v2/{project_id}/search
router = APIRouter(tags=["search"])
@router.post("/search/", response_model=SearchResponse)
async def search(
project_id: ProjectIdPathDep,
query: SearchQuery,
search_service: SearchServiceV2Dep,
entity_service: EntityServiceV2Dep,
page: int = 1,
page_size: int = 10,
):
"""Search across all knowledge and documents in a project.
V2 uses integer project IDs for improved performance and stability.
Args:
project_id: Validated numeric project ID from URL path
query: Search query parameters (text, filters, etc.)
search_service: Search service scoped to project
entity_service: Entity service scoped to project
page: Page number for pagination
page_size: Number of results per page
Returns:
SearchResponse with paginated search results
"""
limit = page_size
offset = (page - 1) * page_size
results = await search_service.search(query, limit=limit, offset=offset)
search_results = await to_search_results(entity_service, results)
return SearchResponse(
results=search_results,
current_page=page,
page_size=page_size,
)
@router.post("/search/reindex")
async def reindex(
project_id: ProjectIdPathDep,
background_tasks: BackgroundTasks,
search_service: SearchServiceV2Dep,
):
"""Recreate and populate the search index for a project.
This is a background operation that rebuilds the search index
from scratch. Useful after bulk updates or if the index becomes
corrupted.
Args:
project_id: Validated numeric project ID from URL path
background_tasks: FastAPI background tasks handler
search_service: Search service scoped to project
Returns:
Status message indicating reindex has been initiated
"""
await search_service.reindex_all(background_tasks=background_tasks)
return {"status": "ok", "message": "Reindex initiated"}
+6 -17
View File
@@ -100,23 +100,6 @@ class BasicMemoryConfig(BaseSettings):
description="Database connection URL. For Postgres, use postgresql+asyncpg://user:pass@host:port/db. If not set, SQLite will use default path.",
)
# Database connection pool configuration (Postgres only)
db_pool_size: int = Field(
default=20,
description="Number of connections to keep in the pool (Postgres only)",
gt=0,
)
db_pool_overflow: int = Field(
default=40,
description="Max additional connections beyond pool_size under load (Postgres only)",
gt=0,
)
db_pool_recycle: int = Field(
default=180,
description="Recycle connections after N seconds to prevent stale connections. Default 180s works well with Neon's ~5 minute scale-to-zero (Postgres only)",
gt=0,
)
# Watch service configuration
sync_delay: int = Field(
default=1000, description="Milliseconds to wait after changes before syncing", gt=0
@@ -149,6 +132,12 @@ class BasicMemoryConfig(BaseSettings):
gt=0,
)
sync_batch_size: int = Field(
default=100,
description="Number of files to process in a single database transaction during sync. Higher values improve performance with remote databases (Postgres) but increase memory usage. Typical values: 100 (conservative), 500 (balanced), 1000 (aggressive).",
gt=0,
)
kebab_filenames: bool = Field(
default=False,
description="Format for generated filenames. False preserves spaces and special chars, True converts them to hyphens for consistency with permalinks",
+19 -40
View File
@@ -33,7 +33,6 @@ class DatabaseType(Enum):
MEMORY = auto()
FILESYSTEM = auto()
POSTGRES = auto()
@classmethod
def get_db_url(
@@ -43,7 +42,7 @@ class DatabaseType(Enum):
Args:
db_path: Path to SQLite database file (ignored for Postgres)
db_type: Type of database (MEMORY, FILESYSTEM, or POSTGRES)
db_type: Type of database (MEMORY or FILESYSTEM)
config: Optional config to check for database backend and URL
Returns:
@@ -53,21 +52,16 @@ class DatabaseType(Enum):
if config is None:
config = ConfigManager().config
# Handle explicit Postgres type
if db_type == cls.POSTGRES:
if not config.database_url:
raise ValueError("DATABASE_URL must be set when using Postgres backend")
logger.info(f"Using Postgres database: {config.database_url}")
return config.database_url
# Check if Postgres backend is configured (for backward compatibility)
# Check if Postgres backend is configured
if config.database_backend == DatabaseBackend.POSTGRES:
if not config.database_url:
raise ValueError("DATABASE_URL must be set when using Postgres backend")
logger.info(f"Using Postgres database: {config.database_url}")
logger.info(
f"Using Postgres database: {config.database_url.split('@')[1] if '@' in config.database_url else config.database_url}"
)
return config.database_url
# SQLite databases
# Default to SQLite
if db_type == cls.MEMORY:
logger.info("Using in-memory SQLite database")
return "sqlite+aiosqlite://"
@@ -190,37 +184,21 @@ def _create_sqlite_engine(db_url: str, db_type: DatabaseType) -> AsyncEngine:
return engine
def _create_postgres_engine(db_url: str, config: BasicMemoryConfig) -> AsyncEngine:
def _create_postgres_engine(db_url: str) -> AsyncEngine:
"""Create Postgres async engine with appropriate configuration.
Args:
db_url: Postgres connection URL (postgresql+asyncpg://...)
config: BasicMemoryConfig with pool settings
Returns:
Configured async engine for Postgres
"""
# Use NullPool connection issues.
# Assume connection pooler like PgBouncer handles connection pooling.
# Postgres with asyncpg - use standard async connection
engine = create_async_engine(
db_url,
echo=False,
poolclass=NullPool, # No pooling - fresh connection per request
connect_args={
# Disable statement cache to avoid issues with prepared statements on reconnect
"statement_cache_size": 0,
# Allow 30s for commands (Neon cold start can take 2-5s, sometimes longer)
"command_timeout": 30,
# Allow 30s for initial connection (Neon wake-up time)
"timeout": 30,
"server_settings": {
"application_name": "basic-memory",
# Statement timeout for queries (30s to allow for cold start)
"statement_timeout": "30s",
},
},
pool_pre_ping=True, # Verify connections before using them
)
logger.debug("Created Postgres engine with NullPool (no connection pooling)")
return engine
@@ -232,7 +210,7 @@ def _create_engine_and_session(
Args:
db_path: Path to database file (used for SQLite, ignored for Postgres)
db_type: Type of database (MEMORY, FILESYSTEM, or POSTGRES)
db_type: Type of database (MEMORY or FILESYSTEM)
Returns:
Tuple of (engine, session_maker)
@@ -242,9 +220,8 @@ def _create_engine_and_session(
logger.debug(f"Creating engine for db_url: {db_url}")
# Delegate to backend-specific engine creation
# Check explicit POSTGRES type first, then config setting
if db_type == DatabaseType.POSTGRES or config.database_backend == DatabaseBackend.POSTGRES:
engine = _create_postgres_engine(db_url, config)
if config.database_backend == DatabaseBackend.POSTGRES:
engine = _create_postgres_engine(db_url)
else:
engine = _create_sqlite_engine(db_url, db_type)
@@ -349,8 +326,13 @@ async def run_migrations(
config.set_main_option("revision_environment", "false")
# Get the correct database URL based on backend configuration
# No URL conversion needed - env.py now handles both async and sync engines
db_url = DatabaseType.get_db_url(app_config.database_path, database_type, app_config)
# For Postgres, Alembic needs synchronous driver (psycopg2), not async (asyncpg)
if app_config.database_backend == DatabaseBackend.POSTGRES:
# Convert asyncpg URL to psycopg2 URL for Alembic
db_url = db_url.replace("postgresql+asyncpg://", "postgresql://")
config.set_main_option("sqlalchemy.url", db_url)
command.upgrade(config, "head")
@@ -366,10 +348,7 @@ async def run_migrations(
# For SQLite: Create FTS5 virtual table
# For Postgres: No-op (tsvector column added by migrations)
# The project_id is not used for init_search_index, so we pass a dummy value
if (
database_type == DatabaseType.POSTGRES
or app_config.database_backend == DatabaseBackend.POSTGRES
):
if app_config.database_backend == DatabaseBackend.POSTGRES:
await PostgresSearchRepository(session_maker, 1).init_search_index()
else:
await SQLiteSearchRepository(session_maker, 1).init_search_index()
+3 -281
View File
@@ -76,34 +76,6 @@ async def get_project_config(
ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)] # pragma: no cover
async def get_project_config_v2(
project_id: "ProjectIdPathDep", project_repository: "ProjectRepositoryDep"
) -> ProjectConfig: # pragma: no cover
"""Get the project config for v2 API (uses integer project_id from path).
Args:
project_id: The validated numeric project ID from the URL path
project_repository: Repository for project operations
Returns:
The resolved project config
Raises:
HTTPException: If project is not found
"""
project_obj = await project_repository.get_by_id(project_id)
if project_obj:
return ProjectConfig(name=project_obj.name, home=pathlib.Path(project_obj.path))
# Not found (this should not happen since ProjectIdPathDep already validates existence)
raise HTTPException( # pragma: no cover
status_code=status.HTTP_404_NOT_FOUND, detail=f"Project with ID {project_id} not found."
)
ProjectConfigV2Dep = Annotated[ProjectConfig, Depends(get_project_config_v2)] # pragma: no cover
## sqlalchemy
@@ -158,38 +130,6 @@ ProjectRepositoryDep = Annotated[ProjectRepository, Depends(get_project_reposito
ProjectPathDep = Annotated[str, Path()] # Use Path dependency to extract from URL
async def validate_project_id(
project_id: int,
project_repository: ProjectRepositoryDep,
) -> int:
"""Validate that a numeric project ID exists in the database.
This is used for v2 API endpoints that take project IDs as integers in the path.
The project_id parameter will be automatically extracted from the URL path by FastAPI.
Args:
project_id: The numeric project ID from the URL path
project_repository: Repository for project operations
Returns:
The validated project ID
Raises:
HTTPException: If project with that ID is not found
"""
project_obj = await project_repository.get_by_id(project_id)
if not project_obj:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Project with ID {project_id} not found.",
)
return project_id
# V2 API: Validated integer project ID from path
ProjectIdPathDep = Annotated[int, Depends(validate_project_id)]
async def get_project_id(
project_repository: ProjectRepositoryDep,
project: ProjectPathDep,
@@ -248,17 +188,6 @@ async def get_entity_repository(
EntityRepositoryDep = Annotated[EntityRepository, Depends(get_entity_repository)]
async def get_entity_repository_v2(
session_maker: SessionMakerDep,
project_id: ProjectIdPathDep,
) -> EntityRepository:
"""Create an EntityRepository instance for v2 API (uses integer project_id from path)."""
return EntityRepository(session_maker, project_id=project_id)
EntityRepositoryV2Dep = Annotated[EntityRepository, Depends(get_entity_repository_v2)]
async def get_observation_repository(
session_maker: SessionMakerDep,
project_id: ProjectIdDep,
@@ -270,19 +199,6 @@ async def get_observation_repository(
ObservationRepositoryDep = Annotated[ObservationRepository, Depends(get_observation_repository)]
async def get_observation_repository_v2(
session_maker: SessionMakerDep,
project_id: ProjectIdPathDep,
) -> ObservationRepository:
"""Create an ObservationRepository instance for v2 API."""
return ObservationRepository(session_maker, project_id=project_id)
ObservationRepositoryV2Dep = Annotated[
ObservationRepository, Depends(get_observation_repository_v2)
]
async def get_relation_repository(
session_maker: SessionMakerDep,
project_id: ProjectIdDep,
@@ -294,17 +210,6 @@ async def get_relation_repository(
RelationRepositoryDep = Annotated[RelationRepository, Depends(get_relation_repository)]
async def get_relation_repository_v2(
session_maker: SessionMakerDep,
project_id: ProjectIdPathDep,
) -> RelationRepository:
"""Create a RelationRepository instance for v2 API."""
return RelationRepository(session_maker, project_id=project_id)
RelationRepositoryV2Dep = Annotated[RelationRepository, Depends(get_relation_repository_v2)]
async def get_search_repository(
session_maker: SessionMakerDep,
project_id: ProjectIdDep,
@@ -320,17 +225,6 @@ async def get_search_repository(
SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)]
async def get_search_repository_v2(
session_maker: SessionMakerDep,
project_id: ProjectIdPathDep,
) -> SearchRepository:
"""Create a SearchRepository instance for v2 API."""
return create_search_repository(session_maker, project_id=project_id)
SearchRepositoryV2Dep = Annotated[SearchRepository, Depends(get_search_repository_v2)]
# ProjectInfoRepository is deprecated and will be removed in a future version.
# Use ProjectRepository instead, which has the same functionality plus more project-specific operations.
@@ -344,13 +238,6 @@ async def get_entity_parser(project_config: ProjectConfigDep) -> EntityParser:
EntityParserDep = Annotated["EntityParser", Depends(get_entity_parser)]
async def get_entity_parser_v2(project_config: ProjectConfigV2Dep) -> EntityParser:
return EntityParser(project_config.home)
EntityParserV2Dep = Annotated["EntityParser", Depends(get_entity_parser_v2)]
async def get_markdown_processor(entity_parser: EntityParserDep) -> MarkdownProcessor:
return MarkdownProcessor(entity_parser)
@@ -358,39 +245,20 @@ async def get_markdown_processor(entity_parser: EntityParserDep) -> MarkdownProc
MarkdownProcessorDep = Annotated[MarkdownProcessor, Depends(get_markdown_processor)]
async def get_markdown_processor_v2(entity_parser: EntityParserV2Dep) -> MarkdownProcessor:
return MarkdownProcessor(entity_parser)
MarkdownProcessorV2Dep = Annotated[MarkdownProcessor, Depends(get_markdown_processor_v2)]
async def get_file_service(
project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
) -> FileService:
file_service = FileService(project_config.home, markdown_processor)
logger.debug(
f"Created FileService for project: {project_config.name}, base_path: {project_config.home} "
f"Creating FileService for project: {project_config.name}, base_path: {project_config.home}"
)
file_service = FileService(project_config.home, markdown_processor)
logger.debug(f"Created FileService for project: {file_service} ")
return file_service
FileServiceDep = Annotated[FileService, Depends(get_file_service)]
async def get_file_service_v2(
project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
) -> FileService:
file_service = FileService(project_config.home, markdown_processor)
logger.debug(
f"Created FileService for project: {project_config.name}, base_path: {project_config.home}"
)
return file_service
FileServiceV2Dep = Annotated[FileService, Depends(get_file_service_v2)]
async def get_entity_service(
entity_repository: EntityRepositoryDep,
observation_repository: ObservationRepositoryDep,
@@ -415,30 +283,6 @@ async def get_entity_service(
EntityServiceDep = Annotated[EntityService, Depends(get_entity_service)]
async def get_entity_service_v2(
entity_repository: EntityRepositoryV2Dep,
observation_repository: ObservationRepositoryV2Dep,
relation_repository: RelationRepositoryV2Dep,
entity_parser: EntityParserV2Dep,
file_service: FileServiceV2Dep,
link_resolver: "LinkResolverV2Dep",
app_config: AppConfigDep,
) -> EntityService:
"""Create EntityService for v2 API."""
return EntityService(
entity_repository=entity_repository,
observation_repository=observation_repository,
relation_repository=relation_repository,
entity_parser=entity_parser,
file_service=file_service,
link_resolver=link_resolver,
app_config=app_config,
)
EntityServiceV2Dep = Annotated[EntityService, Depends(get_entity_service_v2)]
async def get_search_service(
search_repository: SearchRepositoryDep,
entity_repository: EntityRepositoryDep,
@@ -451,18 +295,6 @@ async def get_search_service(
SearchServiceDep = Annotated[SearchService, Depends(get_search_service)]
async def get_search_service_v2(
search_repository: SearchRepositoryV2Dep,
entity_repository: EntityRepositoryV2Dep,
file_service: FileServiceV2Dep,
) -> SearchService:
"""Create SearchService for v2 API."""
return SearchService(search_repository, entity_repository, file_service)
SearchServiceV2Dep = Annotated[SearchService, Depends(get_search_service_v2)]
async def get_link_resolver(
entity_repository: EntityRepositoryDep, search_service: SearchServiceDep
) -> LinkResolver:
@@ -472,15 +304,6 @@ async def get_link_resolver(
LinkResolverDep = Annotated[LinkResolver, Depends(get_link_resolver)]
async def get_link_resolver_v2(
entity_repository: EntityRepositoryV2Dep, search_service: SearchServiceV2Dep
) -> LinkResolver:
return LinkResolver(entity_repository=entity_repository, search_service=search_service)
LinkResolverV2Dep = Annotated[LinkResolver, Depends(get_link_resolver_v2)]
async def get_context_service(
search_repository: SearchRepositoryDep,
entity_repository: EntityRepositoryDep,
@@ -496,22 +319,6 @@ async def get_context_service(
ContextServiceDep = Annotated[ContextService, Depends(get_context_service)]
async def get_context_service_v2(
search_repository: SearchRepositoryV2Dep,
entity_repository: EntityRepositoryV2Dep,
observation_repository: ObservationRepositoryV2Dep,
) -> ContextService:
"""Create ContextService for v2 API."""
return ContextService(
search_repository=search_repository,
entity_repository=entity_repository,
observation_repository=observation_repository,
)
ContextServiceV2Dep = Annotated[ContextService, Depends(get_context_service_v2)]
async def get_sync_service(
app_config: AppConfigDep,
entity_service: EntityServiceDep,
@@ -541,32 +348,6 @@ async def get_sync_service(
SyncServiceDep = Annotated[SyncService, Depends(get_sync_service)]
async def get_sync_service_v2(
app_config: AppConfigDep,
entity_service: EntityServiceV2Dep,
entity_parser: EntityParserV2Dep,
entity_repository: EntityRepositoryV2Dep,
relation_repository: RelationRepositoryV2Dep,
project_repository: ProjectRepositoryDep,
search_service: SearchServiceV2Dep,
file_service: FileServiceV2Dep,
) -> SyncService: # pragma: no cover
"""Create SyncService for v2 API."""
return SyncService(
app_config=app_config,
entity_service=entity_service,
entity_parser=entity_parser,
entity_repository=entity_repository,
relation_repository=relation_repository,
project_repository=project_repository,
search_service=search_service,
file_service=file_service,
)
SyncServiceV2Dep = Annotated[SyncService, Depends(get_sync_service_v2)]
async def get_project_service(
project_repository: ProjectRepositoryDep,
) -> ProjectService:
@@ -589,18 +370,6 @@ async def get_directory_service(
DirectoryServiceDep = Annotated[DirectoryService, Depends(get_directory_service)]
async def get_directory_service_v2(
entity_repository: EntityRepositoryV2Dep,
) -> DirectoryService:
"""Create DirectoryService for v2 API (uses integer project_id from path)."""
return DirectoryService(
entity_repository=entity_repository,
)
DirectoryServiceV2Dep = Annotated[DirectoryService, Depends(get_directory_service_v2)]
# Import
@@ -644,50 +413,3 @@ async def get_memory_json_importer(
MemoryJsonImporterDep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer)]
# V2 Import dependencies
async def get_chatgpt_importer_v2(
project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
) -> ChatGPTImporter:
"""Create ChatGPTImporter with v2 dependencies."""
return ChatGPTImporter(project_config.home, markdown_processor)
ChatGPTImporterV2Dep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer_v2)]
async def get_claude_conversations_importer_v2(
project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
) -> ClaudeConversationsImporter:
"""Create ClaudeConversationsImporter with v2 dependencies."""
return ClaudeConversationsImporter(project_config.home, markdown_processor)
ClaudeConversationsImporterV2Dep = Annotated[
ClaudeConversationsImporter, Depends(get_claude_conversations_importer_v2)
]
async def get_claude_projects_importer_v2(
project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
) -> ClaudeProjectsImporter:
"""Create ClaudeProjectsImporter with v2 dependencies."""
return ClaudeProjectsImporter(project_config.home, markdown_processor)
ClaudeProjectsImporterV2Dep = Annotated[
ClaudeProjectsImporter, Depends(get_claude_projects_importer_v2)
]
async def get_memory_json_importer_v2(
project_config: ProjectConfigV2Dep, markdown_processor: MarkdownProcessorV2Dep
) -> MemoryJsonImporter:
"""Create MemoryJsonImporter with v2 dependencies."""
return MemoryJsonImporter(project_config.home, markdown_processor)
MemoryJsonImporterV2Dep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer_v2)]
@@ -40,13 +40,10 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
chats_imported = 0
for chat in conversations:
# Get name, providing default for unnamed conversations
chat_name = chat.get("name") or f"Conversation {chat.get('uuid', 'untitled')}"
# Convert to entity
entity = self._format_chat_content(
base_path=folder_path,
name=chat_name,
name=chat["name"],
messages=chat["chat_messages"],
created_at=chat["created_at"],
modified_at=chat["updated_at"],
+23 -55
View File
@@ -189,63 +189,35 @@ class EntityParser:
return self.base_path / path
async def parse_file_content(self, absolute_path, file_content):
"""Parse markdown content from file stats.
Delegates to parse_markdown_content() for actual parsing logic.
Exists for backwards compatibility with code that passes file paths.
"""
# Extract file stat info for timestamps
file_stats = absolute_path.stat()
# Delegate to parse_markdown_content with timestamps from file stats
return await self.parse_markdown_content(
file_path=absolute_path,
content=file_content,
mtime=file_stats.st_mtime,
ctime=file_stats.st_ctime,
)
async def parse_markdown_content(
self,
file_path: Path,
content: str,
mtime: Optional[float] = None,
ctime: Optional[float] = None,
) -> EntityMarkdown:
"""Parse markdown content without requiring file to exist on disk.
Useful for parsing content from S3 or other remote sources where the file
is not available locally.
Args:
file_path: Path for metadata (doesn't need to exist on disk)
content: Markdown content as string
mtime: Optional modification time (Unix timestamp)
ctime: Optional creation time (Unix timestamp)
Returns:
EntityMarkdown with parsed content
"""
# Parse frontmatter with proper error handling for malformed YAML
# Parse frontmatter with proper error handling for malformed YAML (issue #185)
try:
post = frontmatter.loads(content)
post = frontmatter.loads(file_content)
except yaml.YAMLError as e:
# Log the YAML parsing error with file context
logger.warning(
f"Failed to parse YAML frontmatter in {file_path}: {e}. "
f"Failed to parse YAML frontmatter in {absolute_path}: {e}. "
f"Treating file as plain markdown without frontmatter."
)
post = frontmatter.Post(content, metadata={})
# Create a post with no frontmatter - treat entire content as markdown
post = frontmatter.Post(file_content, metadata={})
# Normalize frontmatter values
# Extract file stat info
file_stats = absolute_path.stat()
# Normalize frontmatter values to prevent AttributeError on date objects (issue #236)
# PyYAML automatically converts date strings like "2025-10-24" to datetime.date objects
# This normalization converts them back to ISO format strings to ensure compatibility
# with code that expects string values
metadata = normalize_frontmatter_metadata(post.metadata)
# Ensure required fields have defaults
# Ensure required fields have defaults (issue #184, #387)
# Handle title - use default if missing, None/null, empty, or string "None"
title = metadata.get("title")
if not title or title == "None":
metadata["title"] = file_path.stem
metadata["title"] = absolute_path.stem
else:
metadata["title"] = title
# Handle type - use default if missing OR explicitly set to None/null
entity_type = metadata.get("type")
metadata["type"] = entity_type if entity_type is not None else "note"
@@ -253,20 +225,16 @@ class EntityParser:
if tags:
metadata["tags"] = tags
# Parse content for observations and relations
entity_frontmatter = EntityFrontmatter(metadata=metadata)
# frontmatter - use metadata with defaults applied
entity_frontmatter = EntityFrontmatter(
metadata=metadata,
)
entity_content = parse(post.content)
# Use provided timestamps or current time as fallback
now = datetime.now().astimezone()
created = datetime.fromtimestamp(ctime).astimezone() if ctime else now
modified = datetime.fromtimestamp(mtime).astimezone() if mtime else now
return EntityMarkdown(
frontmatter=entity_frontmatter,
content=post.content,
observations=entity_content.observations,
relations=entity_content.relations,
created=created,
modified=modified,
created=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
modified=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
)
+1 -3
View File
@@ -30,9 +30,7 @@ def is_observation(token: Token) -> bool:
# Check for proper observation format: [category] content
match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
# Match proper hashtags (word characters after #), not HTML color codes
# Negative lookbehind ensures # is not preceded by hex digit or equals sign
has_tags = bool(re.search(r"(?<![0-9a-fA-F=])#\w+", content))
has_tags = "#" in content
return bool(match) or has_tags
+2 -7
View File
@@ -129,7 +129,7 @@ class Entity(Base):
return value
def __repr__(self) -> str:
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}', checksum='{self.checksum}')"
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
class Observation(Base):
@@ -162,14 +162,9 @@ class Observation(Base):
We can construct these because observations are always defined in
and owned by a single entity.
Note: Content is truncated to 150 chars to prevent exceeding PostgreSQL's
btree index limit (2704 bytes) for the permalink column.
"""
# Truncate content to prevent permalink overflow in database indexes
content_for_permalink = self.content[:150] if len(self.content) > 150 else self.content
return generate_permalink(
f"{self.entity.permalink}/observations/{self.category}/{content_for_permalink}"
f"{self.entity.permalink}/observations/{self.category}/{self.content}"
)
def __repr__(self) -> str: # pragma: no cover
@@ -1,16 +1,14 @@
"""Repository for managing entities in the knowledge graph."""
from pathlib import Path
from typing import List, Optional, Sequence, Union, Any
from typing import List, Optional, Sequence, Union
import logfire
from loguru import logger
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
from sqlalchemy.engine import Row
from basic_memory import db
from basic_memory.models.knowledge import Entity, Observation, Relation
@@ -33,20 +31,6 @@ class EntityRepository(Repository[Entity]):
"""
super().__init__(session_maker, Entity, project_id=project_id)
@logfire.instrument()
async def get_by_id(self, entity_id: int) -> Optional[Entity]:
"""Get entity by numeric ID.
Args:
entity_id: Numeric entity ID
Returns:
Entity if found, None otherwise
"""
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, entity_id)
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
"""Get entity by permalink.
@@ -56,7 +40,6 @@ class EntityRepository(Repository[Entity]):
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
return await self.find_one(query)
@logfire.instrument()
async def get_by_title(self, title: str) -> Sequence[Entity]:
"""Get entity by title.
@@ -67,7 +50,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
"""Get entity by file_path.
@@ -81,36 +63,38 @@ class EntityRepository(Repository[Entity]):
)
return await self.find_one(query)
@logfire.instrument()
async def get_by_file_paths(
self, session: AsyncSession, file_paths: Sequence[Union[Path, str]]
) -> List[Row[Any]]:
"""Get file paths and checksums for multiple entities (optimized for change detection).
async def get_by_file_paths_batch(
self, file_paths: Sequence[Union[Path, str]]
) -> dict[str, Entity]:
"""Batch fetch entities by file paths with eager-loaded relationships.
Only queries file_path and checksum columns, skips loading full entities and relationships.
This is much faster than loading complete Entity objects when you only need checksums.
Optimized for scan operations - reduces N queries to 1 batched query.
Returns entities with relationships already loaded via selectinload.
Args:
session: Database session to use for the query
file_paths: List of file paths to query
file_paths: List of file paths to fetch entities for
Returns:
List of (file_path, checksum) tuples for matching entities
Dict mapping file_path (as posix string) -> Entity
Only includes entities that exist; missing files are not in dict
"""
if not file_paths:
return []
return {}
# Convert all paths to POSIX strings for consistent comparison
posix_paths = [Path(fp).as_posix() for fp in file_paths]
# Convert all paths to posix strings
posix_paths = [Path(p).as_posix() for p in file_paths]
# Query ONLY file_path and checksum columns (not full Entity objects)
query = select(Entity.file_path, Entity.checksum).where(Entity.file_path.in_(posix_paths))
query = self._add_project_filter(query)
# Batch query with eager loading
query = (
self.select().where(Entity.file_path.in_(posix_paths)).options(*self.get_load_options())
)
result = await session.execute(query)
return list(result.all())
result = await self.execute_query(query)
entities = list(result.scalars().all())
# Return as dict for O(1) lookup
return {e.file_path: e for e in entities}
@logfire.instrument()
async def find_by_checksum(self, checksum: str) -> Sequence[Entity]:
"""Find entities with the given checksum.
@@ -128,36 +112,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def find_by_checksums(self, checksums: Sequence[str]) -> Sequence[Entity]:
"""Find entities with any of the given checksums (batch query for move detection).
This is a batch-optimized version of find_by_checksum() that queries multiple checksums
in a single database query. Used for efficient move detection in cloud indexing.
Performance: For 1000 new files, this makes 1 query vs 1000 individual queries (~100x faster).
Example:
When processing new files, we check if any are actually moved files by finding
entities with matching checksums at different paths.
Args:
checksums: List of file content checksums to search for
Returns:
Sequence of entities with matching checksums (may be empty).
Multiple entities may have the same checksum if files were copied.
"""
if not checksums:
return []
# Query: SELECT * FROM entities WHERE checksum IN (checksum1, checksum2, ...)
query = self.select().where(Entity.checksum.in_(checksums))
# Don't load relationships for move detection - we only need file_path and checksum
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool:
"""Delete entity with the provided file_path.
@@ -178,7 +132,6 @@ class EntityRepository(Repository[Entity]):
selectinload(Entity.incoming_relations).selectinload(Relation.to_entity),
]
@logfire.instrument()
async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
"""Find multiple entities by their permalink.
@@ -197,7 +150,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def upsert_entity(self, entity: Entity) -> Entity:
"""Insert or update entity using simple try/catch with database-level conflict resolution.
@@ -299,7 +251,6 @@ class EntityRepository(Repository[Entity]):
entity = await self._handle_permalink_conflict(entity, session)
return entity
@logfire.instrument()
async def get_all_file_paths(self) -> List[str]:
"""Get all file paths for this project - optimized for deletion detection.
@@ -315,7 +266,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def get_distinct_directories(self) -> List[str]:
"""Extract unique directory paths from file_path column.
@@ -344,7 +294,6 @@ class EntityRepository(Repository[Entity]):
return sorted(directories)
@logfire.instrument()
async def find_by_directory_prefix(self, directory_prefix: str) -> Sequence[Entity]:
"""Find entities whose file_path starts with the given directory prefix.
@@ -377,7 +326,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def _handle_permalink_conflict(self, entity: Entity, session: AsyncSession) -> Entity:
"""Handle permalink conflicts by generating a unique permalink."""
base_permalink = entity.permalink
@@ -422,3 +370,80 @@ class EntityRepository(Repository[Entity]):
# Re-raise if not a foreign key error
raise
return entity
async def upsert_entities(self, entities: List[Entity]) -> List[Entity]:
"""Bulk insert or update multiple entities in a single transaction.
Optimized for batch operations with remote databases (Postgres).
Handles conflicts the same way as upsert_entity() but processes
all entities in one transaction.
Args:
entities: List of entities to upsert
Returns:
List of upserted entities with relationships loaded
Raises:
SyncFatalError: If any entity references a non-existent project_id
"""
if not entities:
return []
async with db.scoped_session(self.session_maker) as session:
# Set project_id on all entities if needed
for entity in entities:
self._set_project_id_if_needed(entity)
# Try to add all entities
for entity in entities:
session.add(entity)
try:
await session.flush()
# Fetch all entities with relationships loaded
file_paths = [e.file_path for e in entities]
query = (
self.select()
.where(Entity.file_path.in_(file_paths))
.options(*self.get_load_options())
)
result = await session.execute(query)
return list(result.scalars().all())
except IntegrityError as e:
# Check for foreign key constraint failures
error_str = str(e)
if (
"FOREIGN KEY constraint failed" in error_str
or "violates foreign key constraint" in error_str
):
from basic_memory.services.exceptions import SyncFatalError
raise SyncFatalError(
"Cannot sync entities: project_id does not exist in database. "
"The project may have been deleted. This sync will be terminated."
) from e
# For other integrity errors (file_path or permalink conflicts),
# rollback and fall back to individual processing
await session.rollback()
# Process each entity individually to handle conflicts properly
logger.debug(
f"Batch upsert failed with IntegrityError, falling back to individual upserts for {len(entities)} entities"
)
result_entities = []
for entity in entities:
try:
upserted = await self.upsert_entity(entity)
result_entities.append(upserted)
except Exception as individual_error:
logger.error(
f"Failed to upsert entity {entity.file_path}: {individual_error}"
)
# Continue with other entities
return result_entities
@@ -2,7 +2,6 @@
from typing import Dict, List, Sequence
import logfire
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
@@ -22,35 +21,30 @@ class ObservationRepository(Repository[Observation]):
"""
super().__init__(session_maker, Observation, project_id=project_id)
@logfire.instrument()
async def find_by_entity(self, entity_id: int) -> Sequence[Observation]:
"""Find all observations for a specific entity."""
query = select(Observation).filter(Observation.entity_id == entity_id)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_context(self, context: str) -> Sequence[Observation]:
"""Find observations with a specific context."""
query = select(Observation).filter(Observation.context == context)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_category(self, category: str) -> Sequence[Observation]:
"""Find observations with a specific context."""
query = select(Observation).filter(Observation.category == category)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def observation_categories(self) -> Sequence[str]:
"""Return a list of all observation categories."""
query = select(Observation.category).distinct()
result = await self.execute_query(query, use_query_options=False)
return result.scalars().all()
@logfire.instrument()
async def find_by_entities(self, entity_ids: List[int]) -> Dict[int, List[Observation]]:
"""Find all observations for multiple entities in a single query.
@@ -76,3 +70,33 @@ class ObservationRepository(Repository[Observation]):
observations_by_entity[obs.entity_id].append(obs)
return observations_by_entity
async def delete_by_entity_ids(self, entity_ids: List[int]) -> int:
"""Delete all observations for multiple entities in a single query.
Optimized for batch operations - deletes observations for many entities
in one database transaction.
Args:
entity_ids: List of entity IDs whose observations should be deleted
Returns:
Number of observations deleted
"""
if not entity_ids:
return 0
from basic_memory import db
async with db.scoped_session(self.session_maker) as session:
# Use bulk delete with IN clause
query = select(Observation).where(Observation.entity_id.in_(entity_ids))
result = await session.execute(query)
observations_to_delete = result.scalars().all()
# Delete all observations
for obs in observations_to_delete:
await session.delete(obs)
await session.flush()
return len(observations_to_delete)
@@ -5,7 +5,6 @@ import re
from datetime import datetime
from typing import List, Optional
import logfire
from loguru import logger
from sqlalchemy import text
@@ -26,7 +25,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
- JSONB containment operators for metadata search
"""
@logfire.instrument()
async def init_search_index(self):
"""Create Postgres table with tsvector column and GIN indexes.
@@ -147,7 +145,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
else:
return cleaned_term
@logfire.instrument()
async def search(
self,
search_text: Optional[str] = None,
@@ -314,69 +311,3 @@ class PostgresSearchRepository(SearchRepositoryBase):
)
return results
@logfire.instrument()
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
"""Index multiple items in a single batch operation using UPSERT.
Uses INSERT ... ON CONFLICT DO UPDATE to handle re-indexing of existing
entities (e.g., during forward reference resolution) without requiring
a separate delete operation. This eliminates race conditions between
delete and insert operations in separate transactions.
Args:
search_index_rows: List of SearchIndexRow objects to index
"""
if not search_index_rows:
return
async with db.scoped_session(self.session_maker) as session:
# When using text() raw SQL, always serialize JSON to string
# Both SQLite (TEXT) and Postgres (JSONB) accept JSON strings in raw SQL
# The database driver/column type will handle conversion
insert_data_list = []
for row in search_index_rows:
insert_data = row.to_insert(serialize_json=True)
insert_data["project_id"] = self.project_id
insert_data_list.append(insert_data)
# Use UPSERT (INSERT ... ON CONFLICT) to handle re-indexing
# Primary key is (id, type, project_id)
# This handles race conditions during forward reference resolution
# where an entity might be re-indexed before the delete commits
# Syntax works for both SQLite 3.24+ and PostgreSQL
await session.execute(
text("""
INSERT INTO search_index (
id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
from_id, to_id, relation_type,
entity_id, category,
created_at, updated_at,
project_id
) VALUES (
:id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
:from_id, :to_id, :relation_type,
:entity_id, :category,
:created_at, :updated_at,
:project_id
)
ON CONFLICT (id, type, project_id) DO UPDATE SET
title = EXCLUDED.title,
content_stems = EXCLUDED.content_stems,
content_snippet = EXCLUDED.content_snippet,
permalink = EXCLUDED.permalink,
file_path = EXCLUDED.file_path,
metadata = EXCLUDED.metadata,
from_id = EXCLUDED.from_id,
to_id = EXCLUDED.to_id,
relation_type = EXCLUDED.relation_type,
entity_id = EXCLUDED.entity_id,
category = EXCLUDED.category,
created_at = EXCLUDED.created_at,
updated_at = EXCLUDED.updated_at
"""),
insert_data_list,
)
logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
await session.commit()
@@ -3,7 +3,6 @@
from pathlib import Path
from typing import Optional, Sequence, Union
import logfire
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -23,7 +22,6 @@ class ProjectRepository(Repository[Project]):
"""Initialize with session maker."""
super().__init__(session_maker, Project)
@logfire.instrument()
async def get_by_name(self, name: str) -> Optional[Project]:
"""Get project by name.
@@ -33,7 +31,6 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.name == name)
return await self.find_one(query)
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> Optional[Project]:
"""Get project by permalink.
@@ -43,7 +40,6 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.permalink == permalink)
return await self.find_one(query)
@logfire.instrument()
async def get_by_path(self, path: Union[Path, str]) -> Optional[Project]:
"""Get project by filesystem path.
@@ -53,33 +49,17 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.path == Path(path).as_posix())
return await self.find_one(query)
@logfire.instrument()
async def get_by_id(self, project_id: int) -> Optional[Project]:
"""Get project by numeric ID.
Args:
project_id: Numeric project ID
Returns:
Project if found, None otherwise
"""
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, project_id)
@logfire.instrument()
async def get_default_project(self) -> Optional[Project]:
"""Get the default project (the one marked as is_default=True)."""
query = self.select().where(Project.is_default.is_not(None))
return await self.find_one(query)
@logfire.instrument()
async def get_active_projects(self) -> Sequence[Project]:
"""Get all active projects."""
query = self.select().where(Project.is_active == True) # noqa: E712
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def set_as_default(self, project_id: int) -> Optional[Project]:
"""Set a project as the default and unset previous default.
@@ -104,7 +84,6 @@ class ProjectRepository(Repository[Project]):
return target_project
return None # pragma: no cover
@logfire.instrument()
async def update_path(self, project_id: int, new_path: str) -> Optional[Project]:
"""Update project path.
@@ -1,9 +1,9 @@
"""Repository for managing Relation objects."""
from sqlalchemy import and_, delete
from typing import Sequence, List, Optional
import logfire
from sqlalchemy import and_, delete, select
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.orm import selectinload, aliased
from sqlalchemy.orm.interfaces import LoaderOption
@@ -25,7 +25,6 @@ class RelationRepository(Repository[Relation]):
"""
super().__init__(session_maker, Relation, project_id=project_id)
@logfire.instrument()
async def find_relation(
self, from_permalink: str, to_permalink: str, relation_type: str
) -> Optional[Relation]:
@@ -47,21 +46,18 @@ class RelationRepository(Repository[Relation]):
)
return await self.find_one(query)
@logfire.instrument()
async def find_by_entities(self, from_id: int, to_id: int) -> Sequence[Relation]:
"""Find all relations between two entities."""
query = select(Relation).where((Relation.from_id == from_id) & (Relation.to_id == to_id))
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_type(self, relation_type: str) -> Sequence[Relation]:
"""Find all relations of a specific type."""
query = select(Relation).filter(Relation.relation_type == relation_type)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def delete_outgoing_relations_from_entity(self, entity_id: int) -> None:
"""Delete outgoing relations for an entity.
@@ -71,14 +67,33 @@ class RelationRepository(Repository[Relation]):
async with db.scoped_session(self.session_maker) as session:
await session.execute(delete(Relation).where(Relation.from_id == entity_id))
@logfire.instrument()
async def delete_outgoing_relations_from_entities(self, entity_ids: List[int]) -> int:
"""Delete outgoing relations for multiple entities in a single query.
Optimized for batch operations - deletes relations for many entities
in one database transaction. Only deletes relations where these entities
are the source (from_id).
Args:
entity_ids: List of entity IDs whose outgoing relations should be deleted
Returns:
Number of relations deleted
"""
if not entity_ids:
return 0
async with db.scoped_session(self.session_maker) as session:
# Use bulk delete with IN clause
result = await session.execute(delete(Relation).where(Relation.from_id.in_(entity_ids)))
return result.rowcount or 0
async def find_unresolved_relations(self) -> Sequence[Relation]:
"""Find all unresolved relations, where to_id is null."""
query = select(Relation).filter(Relation.to_id.is_(None))
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_unresolved_relations_for_entity(self, entity_id: int) -> Sequence[Relation]:
"""Find unresolved relations for a specific entity.
-16
View File
@@ -2,7 +2,6 @@
from typing import Type, Optional, Any, Sequence, TypeVar, List, Dict
import logfire
from loguru import logger
from sqlalchemy import (
select,
@@ -85,7 +84,6 @@ class Repository[T: Base]:
result = await session.execute(query)
return result.scalars().one_or_none()
@logfire.instrument()
async def select_by_ids(self, session: AsyncSession, ids: List[int]) -> Sequence[T]:
"""Select multiple entities by IDs using an existing session."""
query = (
@@ -97,7 +95,6 @@ class Repository[T: Base]:
result = await session.execute(query)
return result.scalars().all()
@logfire.instrument()
async def add(self, model: T) -> T:
"""
Add a model to the repository. This will also add related objects
@@ -124,7 +121,6 @@ class Repository[T: Base]:
)
return found
@logfire.instrument()
async def add_all(self, models: List[T]) -> Sequence[T]:
"""
Add a list of models to the repository. This will also add related objects
@@ -156,7 +152,6 @@ class Repository[T: Base]:
# Add project filter if applicable
return self._add_project_filter(query)
@logfire.instrument()
async def find_all(
self, skip: int = 0, limit: Optional[int] = None, use_load_options: bool = True
) -> Sequence[T]:
@@ -188,7 +183,6 @@ class Repository[T: Base]:
logger.debug(f"Found {len(items)} {self.Model.__name__} records")
return items
@logfire.instrument()
async def find_by_id(self, entity_id: int) -> Optional[T]:
"""Fetch an entity by its unique identifier."""
logger.debug(f"Finding {self.Model.__name__} by ID: {entity_id}")
@@ -196,7 +190,6 @@ class Repository[T: Base]:
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, entity_id)
@logfire.instrument()
async def find_by_ids(self, ids: List[int]) -> Sequence[T]:
"""Fetch multiple entities by their identifiers in a single query."""
logger.debug(f"Finding {self.Model.__name__} by IDs: {ids}")
@@ -204,7 +197,6 @@ class Repository[T: Base]:
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_ids(session, ids)
@logfire.instrument()
async def find_one(self, query: Select[tuple[T]]) -> Optional[T]:
"""Execute a query and retrieve a single record."""
# add in load options
@@ -218,7 +210,6 @@ class Repository[T: Base]:
logger.trace(f"No {self.Model.__name__} found")
return entity
@logfire.instrument()
async def create(self, data: dict) -> T:
"""Create a new record from a model instance."""
logger.debug(f"Creating {self.Model.__name__} from entity_data: {data}")
@@ -250,7 +241,6 @@ class Repository[T: Base]:
)
return return_instance
@logfire.instrument()
async def create_all(self, data_list: List[dict]) -> Sequence[T]:
"""Create multiple records in a single transaction."""
logger.debug(f"Bulk creating {len(data_list)} {self.Model.__name__} instances")
@@ -276,7 +266,6 @@ class Repository[T: Base]:
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
@logfire.instrument()
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
"""Update an entity with the given data."""
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
@@ -306,7 +295,6 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found to update: {entity_id}")
return None
@logfire.instrument()
async def delete(self, entity_id: int) -> bool:
"""Delete an entity from the database."""
logger.debug(f"Deleting {self.Model.__name__}: {entity_id}")
@@ -324,7 +312,6 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found to delete: {entity_id}")
return False
@logfire.instrument()
async def delete_by_ids(self, ids: List[int]) -> int:
"""Delete records matching given IDs."""
logger.debug(f"Deleting {self.Model.__name__} by ids: {ids}")
@@ -340,7 +327,6 @@ class Repository[T: Base]:
logger.debug(f"Deleted {result.rowcount} records")
return result.rowcount
@logfire.instrument()
async def delete_by_fields(self, **filters: Any) -> bool:
"""Delete records matching given field values."""
logger.debug(f"Deleting {self.Model.__name__} by fields: {filters}")
@@ -357,7 +343,6 @@ class Repository[T: Base]:
logger.debug(f"Deleted {result.rowcount} records")
return deleted
@logfire.instrument()
async def count(self, query: Executable | None = None) -> int:
"""Count entities in the database table."""
async with db.scoped_session(self.session_maker) as session:
@@ -379,7 +364,6 @@ class Repository[T: Base]:
logger.debug(f"Counted {count} {self.Model.__name__} records")
return count
@logfire.instrument()
async def execute_query(
self,
query: Executable,
@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any, Dict, List, Optional
import logfire
from loguru import logger
from sqlalchemy import Executable, Result, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -103,7 +102,6 @@ class SearchRepositoryBase(ABC):
"""
pass
@logfire.instrument()
async def index_item(self, search_index_row: SearchIndexRow) -> None:
"""Index or update a single item.
@@ -147,7 +145,6 @@ class SearchRepositoryBase(ABC):
logger.debug(f"indexed row {search_index_row}")
await session.commit()
@logfire.instrument()
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
"""Index multiple items in a single batch operation.
@@ -195,7 +192,6 @@ class SearchRepositoryBase(ABC):
logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
await session.commit()
@logfire.instrument()
async def delete_by_entity_id(self, entity_id: int) -> None:
"""Delete all search index entries for an entity.
@@ -210,7 +206,6 @@ class SearchRepositoryBase(ABC):
)
await session.commit()
@logfire.instrument()
async def delete_by_permalink(self, permalink: str) -> None:
"""Delete a search index entry by permalink.
@@ -225,7 +220,6 @@ class SearchRepositoryBase(ABC):
)
await session.commit()
@logfire.instrument()
async def execute_query(
self,
query: Executable,
@@ -5,7 +5,6 @@ import re
from datetime import datetime
from typing import List, Optional
import logfire
from loguru import logger
from sqlalchemy import text
@@ -26,7 +25,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
- Prefix wildcard matching with *
"""
@logfire.instrument()
async def init_search_index(self):
"""Create FTS5 virtual table for search.
@@ -281,7 +279,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
# For non-Boolean queries, use the single term preparation logic
return self._prepare_single_term(term, is_prefix)
@logfire.instrument()
async def search(
self,
search_text: Optional[str] = None,
-7
View File
@@ -124,7 +124,6 @@ class EntitySummary(BaseModel):
"""Simplified entity representation."""
type: Literal["entity"] = "entity"
entity_id: int # Database ID for v2 API consistency
permalink: Optional[str]
title: str
content: Optional[str] = None
@@ -142,16 +141,12 @@ class RelationSummary(BaseModel):
"""Simplified relation representation."""
type: Literal["relation"] = "relation"
relation_id: int # Database ID for v2 API consistency
entity_id: Optional[int] = None # ID of the entity this relation belongs to
title: str
file_path: str
permalink: str
relation_type: str
from_entity: Optional[str] = None
from_entity_id: Optional[int] = None # ID of source entity
to_entity: Optional[str] = None
to_entity_id: Optional[int] = None # ID of target entity
created_at: Annotated[
datetime, Field(json_schema_extra={"type": "string", "format": "date-time"})
]
@@ -165,8 +160,6 @@ class ObservationSummary(BaseModel):
"""Simplified observation representation."""
type: Literal["observation"] = "observation"
observation_id: int # Database ID for v2 API consistency
entity_id: Optional[int] = None # ID of the entity this observation belongs to
title: str
file_path: str
permalink: str
-1
View File
@@ -173,7 +173,6 @@ class ProjectWatchStatus(BaseModel):
class ProjectItem(BaseModel):
"""Simple representation of a project."""
id: int
name: str
path: str
is_default: bool = False
-5
View File
@@ -97,11 +97,6 @@ class SearchResult(BaseModel):
metadata: Optional[dict] = None
# IDs for v2 API consistency
entity_id: Optional[int] = None # Entity ID (always present for entities)
observation_id: Optional[int] = None # Observation ID (for observation results)
relation_id: Optional[int] = None # Relation ID (for relation results)
# Type-specific fields
category: Optional[str] = None # For observations
from_entity: Optional[Permalink] = None # For relations
-23
View File
@@ -1,23 +0,0 @@
"""V2 API schemas - ID-based entity references."""
from basic_memory.schemas.v2.entity import (
EntityResolveRequest,
EntityResolveResponse,
EntityResponseV2,
MoveEntityRequestV2,
)
from basic_memory.schemas.v2.resource import (
CreateResourceRequest,
UpdateResourceRequest,
ResourceResponse,
)
__all__ = [
"EntityResolveRequest",
"EntityResolveResponse",
"EntityResponseV2",
"MoveEntityRequestV2",
"CreateResourceRequest",
"UpdateResourceRequest",
"ResourceResponse",
]
-96
View File
@@ -1,96 +0,0 @@
"""V2 entity schemas with ID-first design."""
from datetime import datetime
from typing import Dict, List, Literal, Optional
from pydantic import BaseModel, Field, ConfigDict
from basic_memory.schemas.response import ObservationResponse, RelationResponse
class EntityResolveRequest(BaseModel):
"""Request to resolve a string identifier to an entity ID.
Supports resolution of:
- Permalinks (e.g., "specs/search")
- Titles (e.g., "Search Specification")
- File paths (e.g., "specs/search.md")
"""
identifier: str = Field(
...,
description="Entity identifier to resolve (permalink, title, or file path)",
min_length=1,
max_length=500,
)
class EntityResolveResponse(BaseModel):
"""Response from identifier resolution.
Returns the entity ID and associated metadata for the resolved entity.
"""
entity_id: int = Field(..., description="Numeric entity ID (primary identifier)")
permalink: Optional[str] = Field(None, description="Entity permalink")
file_path: str = Field(..., description="Relative file path")
title: str = Field(..., description="Entity title")
resolution_method: Literal["id", "permalink", "title", "path", "search"] = Field(
..., description="How the identifier was resolved"
)
class MoveEntityRequestV2(BaseModel):
"""V2 request schema for moving an entity to a new file location.
In V2 API, the entity ID is provided in the URL path, so this request
only needs the destination path.
"""
destination_path: str = Field(
...,
description="New file path for the entity (relative to project root)",
min_length=1,
max_length=500,
)
class EntityResponseV2(BaseModel):
"""V2 entity response with ID as the primary field.
This response format emphasizes the entity ID as the primary identifier,
with all other fields (permalink, file_path) as secondary metadata.
"""
# ID first - this is the primary identifier in v2
id: int = Field(..., description="Numeric entity ID (primary identifier)")
# Core entity fields
title: str = Field(..., description="Entity title")
entity_type: str = Field(..., description="Entity type")
content_type: str = Field(default="text/markdown", description="Content MIME type")
# Secondary identifiers (for compatibility and convenience)
permalink: Optional[str] = Field(None, description="Entity permalink (may change)")
file_path: str = Field(..., description="Relative file path (may change)")
# Content and metadata
content: Optional[str] = Field(None, description="Entity content")
entity_metadata: Optional[Dict] = Field(None, description="Entity metadata")
# Relationships
observations: List[ObservationResponse] = Field(
default_factory=list, description="Entity observations"
)
relations: List[RelationResponse] = Field(default_factory=list, description="Entity relations")
# Timestamps
created_at: datetime = Field(..., description="Creation timestamp")
updated_at: datetime = Field(..., description="Last update timestamp")
# V2-specific metadata
api_version: Literal["v2"] = Field(
default="v2", description="API version (always 'v2' for this response)"
)
model_config = ConfigDict(from_attributes=True)
-46
View File
@@ -1,46 +0,0 @@
"""V2 resource schemas for file content operations."""
from pydantic import BaseModel, Field
class CreateResourceRequest(BaseModel):
"""Request to create a new resource file.
File path is required for new resources since we need to know where
to create the file.
"""
file_path: str = Field(
...,
description="Path to create the file, relative to project root",
min_length=1,
max_length=500,
)
content: str = Field(..., description="File content to write")
class UpdateResourceRequest(BaseModel):
"""Request to update an existing resource by entity ID.
Only content is required - the file path is already known from the entity.
Optionally can update the file_path to move the file.
"""
content: str = Field(..., description="File content to write")
file_path: str | None = Field(
None,
description="Optional new file path to move the resource",
min_length=1,
max_length=500,
)
class ResourceResponse(BaseModel):
"""Response from resource operations."""
entity_id: int = Field(..., description="Entity ID of the resource")
file_path: str = Field(..., description="File path of the resource")
checksum: str = Field(..., description="File content checksum")
size: int = Field(..., description="File size in bytes")
created_at: float = Field(..., description="Creation timestamp")
modified_at: float = Field(..., description="Modification timestamp")
@@ -4,7 +4,6 @@ from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import List, Optional, Tuple
import logfire
from loguru import logger
from sqlalchemy import text
@@ -86,7 +85,6 @@ class ContextService:
self.entity_repository = entity_repository
self.observation_repository = observation_repository
@logfire.instrument()
async def build_context(
self,
memory_url: Optional[MemoryUrl] = None,
@@ -217,7 +215,6 @@ class ContextService:
# Return the structured ContextResult
return ContextResult(results=context_results, metadata=metadata)
@logfire.instrument()
async def find_related(
self,
type_id_pairs: List[Tuple[str, int]],
+2 -19
View File
@@ -3,11 +3,8 @@
import fnmatch
import logging
import os
from datetime import datetime
from typing import Dict, List, Optional, Sequence
import logfire
from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
from basic_memory.schemas.directory import DirectoryNode
@@ -15,17 +12,6 @@ from basic_memory.schemas.directory import DirectoryNode
logger = logging.getLogger(__name__)
def _mtime_to_datetime(entity: Entity) -> datetime:
"""Convert entity mtime (file modification time) to datetime.
Returns the file's actual modification time, falling back to updated_at
if mtime is not available.
"""
if entity.mtime:
return datetime.fromtimestamp(entity.mtime).astimezone()
return entity.updated_at
class DirectoryService:
"""Service for working with directory trees."""
@@ -37,7 +23,6 @@ class DirectoryService:
"""
self.entity_repository = entity_repository
@logfire.instrument()
async def get_directory_tree(self) -> DirectoryNode:
"""Build a hierarchical directory tree from indexed files."""
@@ -92,7 +77,7 @@ class DirectoryService:
entity_id=file.id,
entity_type=file.entity_type,
content_type=file.content_type,
updated_at=_mtime_to_datetime(file),
updated_at=file.updated_at,
)
# Add to parent directory's children
@@ -105,7 +90,6 @@ class DirectoryService:
# Return the root node with its children
return root_node
@logfire.instrument()
async def get_directory_structure(self) -> DirectoryNode:
"""Build a hierarchical directory structure without file details.
@@ -149,7 +133,6 @@ class DirectoryService:
return root_node
@logfire.instrument()
async def list_directory(
self,
dir_name: str = "/",
@@ -258,7 +241,7 @@ class DirectoryService:
entity_id=file.id,
entity_type=file.entity_type,
content_type=file.content_type,
updated_at=_mtime_to_datetime(file),
updated_at=file.updated_at,
)
# Add to parent directory's children
+1 -22
View File
@@ -7,7 +7,6 @@ import frontmatter
import yaml
from loguru import logger
from sqlalchemy.exc import IntegrityError
import logfire
from basic_memory.config import ProjectConfig, BasicMemoryConfig
from basic_memory.file_utils import (
@@ -53,7 +52,6 @@ class EntityService(BaseService[EntityModel]):
self.link_resolver = link_resolver
self.app_config = app_config
@logfire.instrument()
async def detect_file_path_conflicts(
self, file_path: str, skip_check: bool = False
) -> List[Entity]:
@@ -93,7 +91,6 @@ class EntityService(BaseService[EntityModel]):
return conflicts
@logfire.instrument()
async def resolve_permalink(
self,
file_path: Permalink | Path,
@@ -152,7 +149,6 @@ class EntityService(BaseService[EntityModel]):
return permalink
@logfire.instrument()
async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
"""Create new entity or update existing one.
Returns: (entity, is_new) where is_new is True if a new entity was created
@@ -174,7 +170,6 @@ class EntityService(BaseService[EntityModel]):
# Create new entity
return await self.create_entity(schema), True
@logfire.instrument()
async def create_entity(self, schema: EntitySchema) -> EntityModel:
"""Create a new entity and write to filesystem."""
logger.debug(f"Creating entity: {schema.title}")
@@ -241,7 +236,6 @@ class EntityService(BaseService[EntityModel]):
# Set final checksum to mark complete
return await self.repository.update(entity.id, {"checksum": checksum})
@logfire.instrument()
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
"""Update an entity's content and metadata."""
logger.debug(
@@ -322,7 +316,6 @@ class EntityService(BaseService[EntityModel]):
return entity
@logfire.instrument()
async def delete_entity(self, permalink_or_id: str | int) -> bool:
"""Delete entity and its file."""
logger.debug(f"Deleting entity: {permalink_or_id}")
@@ -352,7 +345,6 @@ class EntityService(BaseService[EntityModel]):
logger.info(f"Entity not found: {permalink_or_id}")
return True # Already deleted
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> EntityModel:
"""Get entity by type and name combination."""
logger.debug(f"Getting entity by permalink: {permalink}")
@@ -361,24 +353,20 @@ class EntityService(BaseService[EntityModel]):
raise EntityNotFoundError(f"Entity not found: {permalink}")
return db_entity
@logfire.instrument()
async def get_entities_by_id(self, ids: List[int]) -> Sequence[EntityModel]:
"""Get specific entities and their relationships."""
logger.debug(f"Getting entities: {ids}")
return await self.repository.find_by_ids(ids)
@logfire.instrument()
async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
"""Get specific nodes and their relationships."""
logger.debug(f"Getting entities permalinks: {permalinks}")
return await self.repository.find_by_permalinks(permalinks)
@logfire.instrument()
async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None:
"""Delete entity by file path."""
await self.repository.delete_by_file_path(str(file_path))
@logfire.instrument()
async def create_entity_from_markdown(
self, file_path: Path, markdown: EntityMarkdown
) -> EntityModel:
@@ -402,7 +390,6 @@ class EntityService(BaseService[EntityModel]):
logger.error(f"Failed to upsert entity for {file_path}: {e}")
raise EntityCreationError(f"Failed to create entity: {str(e)}") from e
@logfire.instrument()
async def update_entity_and_observations(
self, file_path: Path, markdown: EntityMarkdown
) -> EntityModel:
@@ -443,7 +430,6 @@ class EntityService(BaseService[EntityModel]):
db_entity,
)
@logfire.instrument()
async def update_entity_relations(
self,
path: str,
@@ -462,11 +448,8 @@ class EntityService(BaseService[EntityModel]):
import asyncio
# Create tasks for all relation lookups
# Use strict=True to disable fuzzy search - only exact matches should create resolved relations
# This ensures forward references (links to non-existent entities) remain unresolved (to_id=NULL)
lookup_tasks = [
self.link_resolver.resolve_link(rel.target, strict=True)
for rel in markdown.relations
self.link_resolver.resolve_link(rel.target) for rel in markdown.relations
]
# Execute all lookups in parallel
@@ -515,7 +498,6 @@ class EntityService(BaseService[EntityModel]):
return await self.repository.get_by_file_path(path)
@logfire.instrument()
async def edit_entity(
self,
identifier: str,
@@ -573,7 +555,6 @@ class EntityService(BaseService[EntityModel]):
return entity
@logfire.instrument()
def apply_edit_operation(
self,
current_content: str,
@@ -626,7 +607,6 @@ class EntityService(BaseService[EntityModel]):
else:
raise ValueError(f"Unsupported operation: {operation}")
@logfire.instrument()
def replace_section_content(
self, current_content: str, section_header: str, new_content: str
) -> str:
@@ -746,7 +726,6 @@ class EntityService(BaseService[EntityModel]):
return content + "\n" + current_content # pragma: no cover
return content + current_content # pragma: no cover
@logfire.instrument()
async def move_entity(
self,
identifier: str,
-11
View File
@@ -8,7 +8,6 @@ from pathlib import Path
from typing import Any, Dict, Tuple, Union
import aiofiles
import logfire
import yaml
from basic_memory import file_utils
@@ -60,7 +59,6 @@ class FileService:
"""
return self.base_path / entity.file_path
@logfire.instrument()
async def read_entity_content(self, entity: EntityModel) -> str:
"""Get entity's content without frontmatter or structured sections.
@@ -79,7 +77,6 @@ class FileService:
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
@logfire.instrument()
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem.
@@ -92,7 +89,6 @@ class FileService:
path = self.get_entity_path(entity)
await self.delete_file(path)
@logfire.instrument()
async def exists(self, path: FilePath) -> bool:
"""Check if file exists at the provided path.
@@ -119,7 +115,6 @@ class FileService:
logger.error("Failed to check file existence", path=str(path), error=str(e))
raise FileOperationError(f"Failed to check file existence: {e}")
@logfire.instrument()
async def ensure_directory(self, path: FilePath) -> None:
"""Ensure directory exists, creating if necessary.
@@ -147,7 +142,6 @@ class FileService:
logger.error("Failed to create directory", path=str(path), error=str(e))
raise FileOperationError(f"Failed to create directory {path}: {e}")
@logfire.instrument()
async def write_file(self, path: FilePath, content: str) -> str:
"""Write content to file and return checksum.
@@ -191,7 +185,6 @@ class FileService:
logger.exception("File write error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to write file: {e}")
@logfire.instrument()
async def read_file_content(self, path: FilePath) -> str:
"""Read file content using true async I/O with aiofiles.
@@ -227,7 +220,6 @@ class FileService:
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
@logfire.instrument()
async def read_file(self, path: FilePath) -> Tuple[str, str]:
"""Read file and compute checksum using true async I/O.
@@ -270,7 +262,6 @@ class FileService:
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
@logfire.instrument()
async def delete_file(self, path: FilePath) -> None:
"""Delete file if it exists.
@@ -285,7 +276,6 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
full_path.unlink(missing_ok=True)
@logfire.instrument()
async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
"""Update frontmatter fields in a file while preserving all content.
@@ -354,7 +344,6 @@ class FileService:
)
raise FileOperationError(f"Failed to update frontmatter: {e}")
@logfire.instrument()
async def compute_checksum(self, path: FilePath) -> str:
"""Compute checksum for a file using true async I/O.
@@ -7,7 +7,6 @@ to ensure consistent application startup across all entry points.
import asyncio
from pathlib import Path
import logfire
from loguru import logger
from basic_memory import db
@@ -18,7 +17,6 @@ from basic_memory.repository import (
)
@logfire.instrument()
async def initialize_database(app_config: BasicMemoryConfig) -> None:
"""Initialize database with migrations handled automatically by get_or_create_db.
@@ -40,7 +38,6 @@ async def initialize_database(app_config: BasicMemoryConfig) -> None:
# more specific error if the database is actually unusable
@logfire.instrument()
async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
"""Ensure all projects in config.json exist in the projects table and vice versa.
@@ -74,7 +71,6 @@ async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
logger.info("Continuing with initialization despite synchronization error")
@logfire.instrument()
async def initialize_file_sync(
app_config: BasicMemoryConfig,
):
@@ -145,7 +141,6 @@ async def initialize_file_sync(
return None
@logfire.instrument()
async def initialize_app(
app_config: BasicMemoryConfig,
):
@@ -2,7 +2,6 @@
from typing import Optional, Tuple
import logfire
from loguru import logger
from basic_memory.models import Entity
@@ -27,7 +26,6 @@ class LinkResolver:
self.entity_repository = entity_repository
self.search_service = search_service
@logfire.instrument()
async def resolve_link(
self, link_text: str, use_search: bool = True, strict: bool = False
) -> Optional[Entity]:
@@ -8,7 +8,6 @@ from datetime import datetime
from pathlib import Path
from typing import Dict, Optional, Sequence
import logfire
from loguru import logger
from sqlalchemy import text
@@ -82,7 +81,6 @@ class ProjectService:
"""
return os.environ.get("BASIC_MEMORY_PROJECT", self.config_manager.default_project)
@logfire.instrument()
async def list_projects(self) -> Sequence[Project]:
"""List all projects without loading entity relationships.
@@ -92,7 +90,6 @@ class ProjectService:
"""
return await self.repository.find_all(use_load_options=False)
@logfire.instrument()
async def get_project(self, name: str) -> Optional[Project]:
"""Get the file path for a project by name or permalink."""
return await self.repository.get_by_name(name) or await self.repository.get_by_permalink(
@@ -133,7 +130,6 @@ class ProjectService:
# Not nested in either direction
return False
@logfire.instrument()
async def add_project(self, name: str, path: str, set_default: bool = False) -> None:
"""Add a new project to the configuration and database.
@@ -225,7 +221,6 @@ class ProjectService:
logger.info(f"Project '{name}' added at {resolved_path}")
@logfire.instrument()
async def remove_project(self, name: str, delete_notes: bool = False) -> None:
"""Remove a project from configuration and database.
@@ -276,7 +271,6 @@ class ProjectService:
except Exception as e:
logger.warning(f"Failed to delete project directory {project_path}: {e}")
@logfire.instrument()
async def set_default_project(self, name: str) -> None:
"""Set the default project in configuration and database.
@@ -301,7 +295,6 @@ class ProjectService:
logger.info(f"Project '{name}' set as default in configuration and database")
@logfire.instrument()
async def _ensure_single_default_project(self) -> None:
"""Ensure only one project has is_default=True.
@@ -343,7 +336,6 @@ class ProjectService:
f"Set '{config_default}' as default project (was missing)"
) # pragma: no cover
@logfire.instrument()
async def synchronize_projects(self) -> None: # pragma: no cover
"""Synchronize projects between database and configuration.
@@ -428,7 +420,6 @@ class ProjectService:
logger.info("Project synchronization complete")
@logfire.instrument()
async def move_project(self, name: str, new_path: str) -> None:
"""Move a project to a new location.
@@ -470,7 +461,6 @@ class ProjectService:
self.config_manager.save_config(config)
raise ValueError(f"Project '{name}' not found in database")
@logfire.instrument()
async def update_project( # pragma: no cover
self, name: str, updated_path: Optional[str] = None, is_active: Optional[bool] = None
) -> None:
@@ -530,7 +520,6 @@ class ProjectService:
f"Changed default project to '{new_default.name}' as '{name}' was deactivated"
)
@logfire.instrument()
async def get_project_info(self, project_name: Optional[str] = None) -> ProjectInfoResponse:
"""Get comprehensive information about the specified Basic Memory project.
@@ -598,7 +587,6 @@ class ProjectService:
system=system,
)
@logfire.instrument()
async def get_statistics(self, project_id: int) -> ProjectStatistics:
"""Get statistics about the specified project.
@@ -715,7 +703,6 @@ class ProjectService:
isolated_entities=isolated_count,
)
@logfire.instrument()
async def get_activity_metrics(self, project_id: int) -> ActivityMetrics:
"""Get activity metrics for the specified project.
+10 -60
View File
@@ -4,7 +4,6 @@ import ast
from datetime import datetime
from typing import List, Optional, Set
import logfire
from dateparser import parse
from fastapi import BackgroundTasks
from loguru import logger
@@ -16,21 +15,6 @@ from basic_memory.repository.search_repository import SearchRepository, SearchIn
from basic_memory.schemas.search import SearchQuery, SearchItemType
from basic_memory.services import FileService
# Maximum size for content_stems field to stay under Postgres's 8KB index row limit.
# We use 6000 characters to leave headroom for other indexed columns and overhead.
MAX_CONTENT_STEMS_SIZE = 6000
def _mtime_to_datetime(entity: Entity) -> datetime:
"""Convert entity mtime (file modification time) to datetime.
Returns the file's actual modification time, falling back to updated_at
if mtime is not available.
"""
if entity.mtime:
return datetime.fromtimestamp(entity.mtime).astimezone()
return entity.updated_at
class SearchService:
"""Service for search operations.
@@ -51,12 +35,10 @@ class SearchService:
self.entity_repository = entity_repository
self.file_service = file_service
@logfire.instrument()
async def init_search_index(self):
"""Create FTS5 virtual table if it doesn't exist."""
await self.repository.init_search_index()
@logfire.instrument()
async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
"""Reindex all content from database."""
@@ -73,7 +55,6 @@ class SearchService:
logger.info("Reindex complete")
@logfire.instrument()
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
"""Search across all indexed content.
@@ -171,33 +152,28 @@ class SearchService:
return [] # pragma: no cover
@logfire.instrument()
async def index_entity(
self,
entity: Entity,
background_tasks: Optional[BackgroundTasks] = None,
content: str | None = None,
) -> None:
if background_tasks:
background_tasks.add_task(self.index_entity_data, entity, content)
background_tasks.add_task(self.index_entity_data, entity)
else:
await self.index_entity_data(entity, content)
await self.index_entity_data(entity)
@logfire.instrument()
async def index_entity_data(
self,
entity: Entity,
content: str | None = None,
) -> None:
# delete all search index data associated with entity
await self.repository.delete_by_entity_id(entity_id=entity.id)
# reindex
await self.index_entity_markdown(
entity, content
entity
) if entity.is_markdown else await self.index_entity_file(entity)
@logfire.instrument()
async def index_entity_file(
self,
entity: Entity,
@@ -215,23 +191,17 @@ class SearchService:
"entity_type": entity.entity_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
updated_at=entity.updated_at,
project_id=entity.project_id,
)
)
@logfire.instrument()
async def index_entity_markdown(
self,
entity: Entity,
content: str | None = None,
) -> None:
"""Index an entity and all its observations and relations.
Args:
entity: The entity to index
content: Optional pre-loaded content (avoids file read). If None, will read from file.
Indexing structure:
1. Entities
- permalink: direct from entity (e.g., "specs/search")
@@ -260,9 +230,7 @@ class SearchService:
title_variants = self._generate_variants(entity.title)
content_stems.extend(title_variants)
# Use provided content or read from file
if content is None:
content = await self.file_service.read_entity_content(entity)
content = await self.file_service.read_entity_content(entity)
if content:
content_stems.append(content)
content_snippet = f"{content[:250]}"
@@ -279,10 +247,6 @@ class SearchService:
entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
# Truncate to stay under Postgres's 8KB index row limit
if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE:
entity_content_stems = entity_content_stems[:MAX_CONTENT_STEMS_SIZE]
# Add entity row
rows_to_index.append(
SearchIndexRow(
@@ -298,28 +262,17 @@ class SearchService:
"entity_type": entity.entity_type,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
updated_at=entity.updated_at,
project_id=entity.project_id,
)
)
# Add observation rows - dedupe by permalink to avoid unique constraint violations
# Two observations with same entity/category/content generate identical permalinks
seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
# Add observation rows
for obs in entity.observations:
obs_permalink = obs.permalink
if obs_permalink in seen_permalinks:
logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
continue
seen_permalinks.add(obs_permalink)
# Index with parent entity's file path since that's where it's defined
obs_content_stems = "\n".join(
p for p in self._generate_variants(obs.content) if p and p.strip()
)
# Truncate to stay under Postgres's 8KB index row limit
if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE:
obs_content_stems = obs_content_stems[:MAX_CONTENT_STEMS_SIZE]
rows_to_index.append(
SearchIndexRow(
id=obs.id,
@@ -327,7 +280,7 @@ class SearchService:
title=f"{obs.category}: {obs.content[:100]}...",
content_stems=obs_content_stems,
content_snippet=obs.content,
permalink=obs_permalink,
permalink=obs.permalink,
file_path=entity.file_path,
category=obs.category,
entity_id=entity.id,
@@ -335,7 +288,7 @@ class SearchService:
"tags": obs.tags,
},
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
updated_at=entity.updated_at,
project_id=entity.project_id,
)
)
@@ -365,7 +318,7 @@ class SearchService:
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at,
updated_at=_mtime_to_datetime(entity),
updated_at=entity.updated_at,
project_id=entity.project_id,
)
)
@@ -373,17 +326,14 @@ class SearchService:
# Batch insert all rows at once
await self.repository.bulk_index_items(rows_to_index)
@logfire.instrument()
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
await self.repository.delete_by_permalink(permalink)
@logfire.instrument()
async def delete_by_entity_id(self, entity_id: int):
"""Delete an item from the search index."""
await self.repository.delete_by_entity_id(entity_id)
@logfire.instrument()
async def handle_delete(self, entity: Entity):
"""Handle complete entity deletion from search index including observations and relations.
+346 -49
View File
@@ -31,6 +31,7 @@ from basic_memory.services import EntityService, FileService
from basic_memory.services.exceptions import SyncFatalError
from basic_memory.services.link_resolver import LinkResolver
from basic_memory.services.search_service import SearchService
from basic_memory.sync.utils import chunks
# Circuit breaker configuration
MAX_CONSECUTIVE_FAILURES = 3
@@ -299,38 +300,139 @@ class SyncService:
for path in report.deleted:
await self.handle_delete(path)
# then new and modified
with logfire.span("process_new_files", new_count=len(report.new)):
for path in report.new:
entity, _ = await self.sync_file(path, new=True)
# then new and modified - process in batches for better performance
batch_size = self.app_config.sync_batch_size
logger.debug(f"Using batch size of {batch_size} for file processing")
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
with logfire.span("process_new_files", new_count=len(report.new)):
# Convert set to list for batching
new_files_list = list(report.new)
for batch in chunks(new_files_list, batch_size):
logger.debug(f"Processing batch of {len(batch)} new files")
# Separate markdown and non-markdown files
markdown_files = [p for p in batch if self.file_service.is_markdown(p)]
regular_files = [p for p in batch if not self.file_service.is_markdown(p)]
# Batch process markdown files
if markdown_files:
try:
batch_results = await self.sync_markdown_batch(markdown_files, new=True)
# Track skipped files
for path, (entity, _) in zip(markdown_files, batch_results):
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
except SyncFatalError:
# Re-raise fatal errors immediately
raise
except Exception as e:
# Batch method raised an exception - record failure for all files in batch
logger.error(f"Batch sync failed for {len(markdown_files)} files: {e}")
for path in markdown_files:
await self._record_failure(path, str(e))
# Track skipped files
if await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
# Process regular files individually (they're already fast)
for path in regular_files:
entity, _ = await self.sync_file(path, new=True)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
)
with logfire.span("process_modified_files", modified_count=len(report.modified)):
for path in report.modified:
entity, _ = await self.sync_file(path, new=False)
# Convert set to list for batching
modified_files_list = list(report.modified)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
for batch in chunks(modified_files_list, batch_size):
logger.debug(f"Processing batch of {len(batch)} modified files")
# Separate markdown and non-markdown files
markdown_files = [p for p in batch if self.file_service.is_markdown(p)]
regular_files = [p for p in batch if not self.file_service.is_markdown(p)]
# Batch process markdown files
if markdown_files:
try:
batch_results = await self.sync_markdown_batch(markdown_files, new=False)
# Track skipped files
for path, (entity, _) in zip(markdown_files, batch_results):
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
except SyncFatalError:
# Re-raise fatal errors immediately
raise
except Exception as e:
# Batch method raised an exception - record failure for all files in batch
logger.error(f"Batch sync failed for {len(markdown_files)} files: {e}")
for path in markdown_files:
await self._record_failure(path, str(e))
# Track skipped files
if await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
# Process regular files individually (they're already fast)
for path in regular_files:
entity, _ = await self.sync_file(path, new=False)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
)
# Only resolve relations if there were actual changes
# If no files changed, no new unresolved relations could have been created
@@ -484,6 +586,12 @@ class SyncService:
logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison")
# Optimization: Batch fetch all entities for files being scanned
# This reduces N queries to 1 batch query (massive performance win for remote DBs)
logger.debug(f"Batch fetching entities for {len(file_paths_to_scan)} files")
entities_by_path = await self.entity_repository.get_by_file_paths_batch(file_paths_to_scan)
logger.debug(f"Found {len(entities_by_path)} existing entities in database")
for rel_path in file_paths_to_scan:
scanned_paths.add(rel_path)
@@ -495,8 +603,8 @@ class SyncService:
stat_info = abs_path.stat()
# Indexed lookup - single file query (not full table scan)
db_entity = await self.entity_repository.get_by_file_path(rel_path)
# O(1) dict lookup instead of database query
db_entity = entities_by_path.get(rel_path)
if db_entity is None:
# New file - need checksum for move detection
@@ -737,6 +845,206 @@ class SyncService:
# Return the final checksum to ensure everything is consistent
return entity, final_checksum
@logfire.instrument()
async def sync_markdown_batch(
self, paths: List[str], new: bool = True
) -> List[Tuple[Optional[Entity], str]]:
"""Sync multiple markdown files in a single batch operation.
Optimized for remote databases (Postgres) - reduces N queries to 1 batch query.
Parses all files first, then does all database operations in one transaction.
Args:
paths: List of paths to markdown files
new: Whether these are new files
Returns:
List of tuples (entity, checksum) for each file
"""
from basic_memory.markdown.utils import entity_model_from_markdown
if not paths:
return []
logger.debug(f"Batch syncing {len(paths)} markdown files (new={new})")
# Phase 1: Parse all files (no DB operations)
parsed_files = []
for path in paths:
# Check if file should be skipped due to repeated failures (circuit breaker)
if await self._should_skip_file(path):
logger.warning(f"Skipping file in batch due to repeated failures: {path}")
parsed_files.append(None)
continue
try:
file_content = await self.file_service.read_file_content(path)
file_contains_frontmatter = has_frontmatter(file_content)
# Get file timestamps for tracking modification times
file_stats = self.file_service.file_stats(path)
created = datetime.fromtimestamp(file_stats.st_ctime).astimezone()
modified = datetime.fromtimestamp(file_stats.st_mtime).astimezone()
# Parse markdown to get entity structure
entity_markdown = await self.entity_parser.parse_file(path)
# Resolve permalink if needed (skip conflict checks during batch)
permalink = entity_markdown.frontmatter.permalink
if file_contains_frontmatter and not self.app_config.disable_permalinks:
permalink = await self.entity_service.resolve_permalink(
path, markdown=entity_markdown, skip_conflict_check=True
)
# If permalink changed, update the file
if permalink != entity_markdown.frontmatter.permalink:
logger.info(
f"Updating permalink for path: {path}, "
f"old_permalink: {entity_markdown.frontmatter.permalink}, "
f"new_permalink: {permalink}"
)
entity_markdown.frontmatter.metadata["permalink"] = permalink
await self.file_service.update_frontmatter(path, {"permalink": permalink})
# Convert to entity model (without saving to DB yet)
entity_model = entity_model_from_markdown(Path(path), entity_markdown)
entity_model.checksum = None # Will be set after relations are resolved
parsed_files.append(
{
"path": path,
"entity_model": entity_model,
"entity_markdown": entity_markdown,
"created": created,
"modified": modified,
"mtime": file_stats.st_mtime,
"size": file_stats.st_size,
}
)
except Exception as e:
# Check if this is a fatal error (or caused by one)
# Fatal errors like project deletion should terminate sync immediately
if isinstance(e, SyncFatalError) or isinstance(e.__cause__, SyncFatalError):
logger.error(
f"Fatal sync error encountered during batch parse, terminating sync: path={path}"
)
raise
# Otherwise treat as recoverable file-level error
logger.error(f"Failed to parse file in batch: path={path}, error={e}")
# Track failure for circuit breaker
await self._record_failure(path, str(e))
parsed_files.append(None) # Mark as failed
# Phase 2: Batch database operations
# Filter out failed parses
valid_files = [f for f in parsed_files if f is not None]
if not valid_files:
logger.warning("No valid files to sync in batch")
return [(None, "") for _ in paths]
# If this is a new batch, upsert all entities at once
if new:
entities_to_upsert = [f["entity_model"] for f in valid_files]
logger.debug(f"Batch upserting {len(entities_to_upsert)} new entities")
upserted_entities = await self.entity_repository.upsert_entities(entities_to_upsert)
# Create lookup by file_path for O(1) access
entities_by_path = {e.file_path: e for e in upserted_entities}
# If updating existing entities, we need to handle observations/relations differently
else:
logger.debug(f"Batch updating {len(valid_files)} existing entities")
# For updates, we need to:
# 1. Get existing entities
# 2. Delete old observations/relations
# 3. Upsert updated entities
file_paths = [f["path"] for f in valid_files]
existing_entities = await self.entity_repository.get_by_file_paths_batch(file_paths)
# Delete old observations and relations in batch
entity_ids = [e.id for e in existing_entities.values() if e.id]
if entity_ids:
await self.entity_service.observation_repository.delete_by_entity_ids(entity_ids)
await self.relation_repository.delete_outgoing_relations_from_entities(entity_ids)
# Upsert all updated entities
entities_to_upsert = [f["entity_model"] for f in valid_files]
upserted_entities = await self.entity_repository.upsert_entities(entities_to_upsert)
# Create lookup by file_path
entities_by_path = {e.file_path: e for e in upserted_entities}
# Phase 3: Post-processing (relations, checksums, search index)
results = []
for i, path in enumerate(paths):
parsed_file = parsed_files[i]
# Skip failed files
if parsed_file is None:
results.append((None, ""))
continue
entity = entities_by_path.get(parsed_file["path"])
if entity is None:
logger.error(f"Entity not found after upsert: {parsed_file['path']}")
results.append((None, ""))
continue
try:
# Update relations for this entity
entity_with_relations = await self.entity_service.update_entity_relations(
parsed_file["path"], parsed_file["entity_markdown"]
)
# Compute final checksum after relations are resolved
final_checksum = await self.file_service.compute_checksum(parsed_file["path"])
# Update checksum, timestamps, and file metadata
await self.entity_repository.update(
entity.id,
{
"checksum": final_checksum,
"created_at": parsed_file["created"],
"updated_at": parsed_file["modified"],
"mtime": parsed_file["mtime"],
"size": parsed_file["size"],
},
)
# Index for search
await self.search_service.index_entity(entity_with_relations)
# Clear failure tracking on successful sync
self._clear_failure(parsed_file["path"])
results.append((entity_with_relations, final_checksum))
logger.debug(
f"Batch sync completed for file: path={parsed_file['path']}, "
f"entity_id={entity.id}, checksum={final_checksum[:8]}"
)
except Exception as e:
# Check if this is a fatal error
if isinstance(e, SyncFatalError) or isinstance(e.__cause__, SyncFatalError):
logger.error(
f"Fatal sync error during post-processing, terminating sync: path={parsed_file['path']}"
)
raise
# Otherwise treat as recoverable file-level error
logger.error(
f"Failed to complete post-processing for file: path={parsed_file['path']}, error={e}"
)
await self._record_failure(parsed_file["path"], str(e))
results.append((None, ""))
continue
return results
@logfire.instrument()
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a non-markdown file with basic tracking.
@@ -1026,27 +1334,16 @@ class SyncService:
"to_name": resolved_entity.title,
},
)
# update search index only on successful resolution
await self.search_service.index_entity(resolved_entity)
except IntegrityError:
# IntegrityError means a relation with this (from_id, to_id, relation_type)
# already exists. The UPDATE was rolled back, so our unresolved relation
# (to_id=NULL) still exists in the database. We delete it because:
# 1. It's redundant - a resolved relation already captures this relationship
# 2. If we don't delete it, future syncs will try to resolve it again
# and get the same IntegrityError
except IntegrityError: # pragma: no cover
logger.debug(
"Deleting duplicate unresolved relation "
"Ignoring duplicate relation "
f"relation_id={relation.id} "
f"from_id={relation.from_id} "
f"to_name={relation.to_name} "
f"resolved_to_id={resolved_entity.id}"
f"to_name={relation.to_name}"
)
try:
await self.relation_repository.delete(relation.id)
except Exception as e:
# Log but don't fail - the relation may have been deleted already
logger.debug(f"Could not delete duplicate relation {relation.id}: {e}")
# update search index
await self.search_service.index_entity(resolved_entity)
async def _quick_count_files(self, directory: Path) -> int:
"""Fast file count using find command.
@@ -1145,13 +1442,13 @@ class SyncService:
async def _scan_directory_full(self, directory: Path) -> List[str]:
"""Full directory scan returning all file paths.
Uses scan_directory() which respects .bmignore patterns.
Uses scan_directory() which respects .bmignore patterns.
Args:
directory: Directory to scan
Args:
directory: Directory to scan
Returns:
List of relative file paths (respects .bmignore)
List of relative file paths (respects .bmignore)
"""
file_paths = []
async for file_path_str, _ in self.scan_directory(directory):
+23
View File
@@ -0,0 +1,23 @@
"""Utilities for sync operations."""
from typing import Iterator, List, TypeVar
T = TypeVar("T")
def chunks(items: List[T], size: int) -> Iterator[List[T]]:
"""Split a list into chunks of specified size.
Args:
items: List of items to chunk
size: Size of each chunk
Yields:
Lists of items, each of specified size (last chunk may be smaller)
Example:
>>> list(chunks([1, 2, 3, 4, 5], 2))
[[1, 2], [3, 4], [5]]
"""
for i in range(0, len(items), size):
yield items[i : i + size]
@@ -77,8 +77,7 @@ async def test_create_project_basic_operation(mcp_server, app, test_project):
assert "test-new-project" in create_text
assert "Project Details:" in create_text
assert "Name: test-new-project" in create_text
# Check path contains project name (platform-independent)
assert "Path:" in create_text and "test-new-project" in create_text
assert "Path: /tmp/test-new-project" in create_text
assert "Project is now available for use" in create_text
# Verify project appears in project list
@@ -437,7 +437,9 @@ async def test_write_note_project_path_validation(mcp_server, app, test_project)
project_with_tilde = ProjectItem(
id=1,
name="Test BiSync", # Name differs from path structure
description="Test",
path="~/Documents/Test BiSync", # Path with tilde
is_active=True,
is_default=False,
)
@@ -252,7 +252,6 @@ async def test_benchmark_sync_100_files(app_config, project_config, config_manag
@pytest.mark.benchmark
@pytest.mark.asyncio
@pytest.mark.skip
async def test_benchmark_sync_500_files(app_config, project_config, config_manager):
"""Benchmark: Sync 500 files (medium repository)."""
results = await run_sync_benchmark(
@@ -269,7 +268,6 @@ async def test_benchmark_sync_500_files(app_config, project_config, config_manag
@pytest.mark.benchmark
@pytest.mark.asyncio
@pytest.mark.slow
@pytest.mark.skip
async def test_benchmark_sync_1000_files(app_config, project_config, config_manager):
"""Benchmark: Sync 1000 files (large repository).
@@ -289,7 +287,6 @@ async def test_benchmark_sync_1000_files(app_config, project_config, config_mana
@pytest.mark.benchmark
@pytest.mark.asyncio
@pytest.mark.skip
async def test_benchmark_resync_no_changes(app_config, project_config, config_manager):
"""Benchmark: Re-sync with no changes (should be fast).
@@ -18,7 +18,6 @@ def template_loader():
def entity_summary():
"""Create a sample EntitySummary for testing."""
return EntitySummary(
entity_id=1,
title="Test Entity",
permalink="test/entity",
type=SearchItemType.ENTITY,
@@ -35,8 +34,6 @@ def context_with_results(entity_summary):
# Create an observation for the entity
observation = ObservationSummary(
observation_id=1,
entity_id=1,
title="Test Observation",
permalink="test/entity/observations/1",
category="test",
-1
View File
@@ -1 +0,0 @@
"""V2 API tests."""
-21
View File
@@ -1,21 +0,0 @@
"""Fixtures for V2 API tests."""
import pytest
from basic_memory.models import Project
@pytest.fixture
def v2_project_url(test_project: Project) -> str:
"""Create a URL prefix for v2 project-scoped routes using project ID.
This helps tests generate the correct URL for v2 project-scoped routes
which use integer project IDs instead of permalinks.
"""
return f"/v2/projects/{test_project.id}"
@pytest.fixture
def v2_projects_url() -> str:
"""Base URL for v2 project management endpoints."""
return "/v2/projects"
-129
View File
@@ -1,129 +0,0 @@
"""Tests for V2 directory API routes (ID-based endpoints)."""
import pytest
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.schemas.directory import DirectoryNode
@pytest.mark.asyncio
async def test_get_directory_tree(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test getting directory tree via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/tree")
assert response.status_code == 200
tree = DirectoryNode.model_validate(response.json())
assert tree.type == "directory"
@pytest.mark.asyncio
async def test_get_directory_structure(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test getting directory structure (folders only) via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/structure")
assert response.status_code == 200
structure = DirectoryNode.model_validate(response.json())
assert structure.type == "directory"
# Structure should only contain directories, not files
if structure.children:
for child in structure.children:
assert child.type == "directory"
@pytest.mark.asyncio
async def test_list_directory_default(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test listing directory contents with default parameters via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/list")
assert response.status_code == 200
nodes = response.json()
assert isinstance(nodes, list)
@pytest.mark.asyncio
async def test_list_directory_with_depth(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test listing directory with custom depth via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/list?depth=2")
assert response.status_code == 200
nodes = response.json()
assert isinstance(nodes, list)
@pytest.mark.asyncio
async def test_list_directory_with_glob(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test listing directory with file name glob filter via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/list?file_name_glob=*.md")
assert response.status_code == 200
nodes = response.json()
assert isinstance(nodes, list)
# All file nodes should have .md extension
for node in nodes:
if node.get("type") == "file":
assert node.get("path", "").endswith(".md")
@pytest.mark.asyncio
async def test_list_directory_with_custom_path(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test listing a specific directory path via v2 endpoint."""
response = await client.get(f"{v2_project_url}/directory/list?dir_name=/")
assert response.status_code == 200
nodes = response.json()
assert isinstance(nodes, list)
@pytest.mark.asyncio
async def test_directory_invalid_project_id(
client: AsyncClient,
):
"""Test directory endpoints with invalid project ID return 404."""
# Test tree endpoint
response = await client.get("/v2/projects/999999/directory/tree")
assert response.status_code == 404
# Test structure endpoint
response = await client.get("/v2/projects/999999/directory/structure")
assert response.status_code == 404
# Test list endpoint
response = await client.get("/v2/projects/999999/directory/list")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_v2_directory_endpoints_use_project_id_not_name(
client: AsyncClient, test_project: Project
):
"""Verify v2 directory endpoints require project ID, not name."""
# Try using project name instead of ID - should fail
response = await client.get(f"/v2/projects/{test_project.name}/directory/tree")
# Should get validation error or 404 because name is not a valid integer
assert response.status_code in [404, 422]
-530
View File
@@ -1,530 +0,0 @@
"""Tests for V2 importer API routes (ID-based endpoints)."""
import json
from pathlib import Path
import pytest
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.schemas.importer import (
ChatImportResult,
EntityImportResult,
ProjectImportResult,
)
@pytest.fixture
def chatgpt_json_content():
"""Sample ChatGPT conversation data for testing."""
return [
{
"title": "Test Conversation",
"create_time": 1736616594.24054,
"update_time": 1736616603.164995,
"mapping": {
"root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]},
"msg1": {
"id": "msg1",
"message": {
"id": "msg1",
"author": {"role": "user", "name": None, "metadata": {}},
"create_time": 1736616594.24054,
"content": {
"content_type": "text",
"parts": ["Hello, this is a test message"],
},
"status": "finished_successfully",
"metadata": {},
},
"parent": "root",
"children": ["msg2"],
},
"msg2": {
"id": "msg2",
"message": {
"id": "msg2",
"author": {"role": "assistant", "name": None, "metadata": {}},
"create_time": 1736616603.164995,
"content": {"content_type": "text", "parts": ["This is a test response"]},
"status": "finished_successfully",
"metadata": {},
},
"parent": "msg1",
"children": [],
},
},
}
]
@pytest.fixture
def claude_conversations_json_content():
"""Sample Claude conversations data for testing."""
return [
{
"uuid": "test-uuid",
"name": "Test Conversation",
"created_at": "2025-01-05T20:55:32.499880+00:00",
"updated_at": "2025-01-05T20:56:39.477600+00:00",
"chat_messages": [
{
"uuid": "msg-1",
"text": "Hello, this is a test",
"sender": "human",
"created_at": "2025-01-05T20:55:32.499880+00:00",
"content": [{"type": "text", "text": "Hello, this is a test"}],
},
{
"uuid": "msg-2",
"text": "Response to test",
"sender": "assistant",
"created_at": "2025-01-05T20:55:40.123456+00:00",
"content": [{"type": "text", "text": "Response to test"}],
},
],
}
]
@pytest.fixture
def claude_projects_json_content():
"""Sample Claude projects data for testing."""
return [
{
"uuid": "test-uuid",
"name": "Test Project",
"created_at": "2025-01-05T20:55:32.499880+00:00",
"updated_at": "2025-01-05T20:56:39.477600+00:00",
"prompt_template": "# Test Prompt\n\nThis is a test prompt.",
"docs": [
{
"uuid": "doc-uuid-1",
"filename": "Test Document",
"content": "# Test Document\n\nThis is test content.",
"created_at": "2025-01-05T20:56:39.477600+00:00",
},
{
"uuid": "doc-uuid-2",
"filename": "Another Document",
"content": "# Another Document\n\nMore test content.",
"created_at": "2025-01-05T20:56:39.477600+00:00",
},
],
}
]
@pytest.fixture
def memory_json_content():
"""Sample memory.json data for testing."""
return [
{
"type": "entity",
"name": "test_entity",
"entityType": "test",
"observations": ["Test observation 1", "Test observation 2"],
},
{
"type": "relation",
"from": "test_entity",
"to": "related_entity",
"relationType": "test_relation",
},
]
async def create_test_upload_file(tmp_path, content):
"""Create a test file for upload."""
file_path = tmp_path / "test_import.json"
with open(file_path, "w", encoding="utf-8") as f:
json.dump(content, f)
return file_path
@pytest.mark.asyncio
async def test_import_chatgpt(
project_config,
client: AsyncClient,
tmp_path,
chatgpt_json_content,
file_service,
v2_project_url: str,
):
"""Test importing ChatGPT conversations via v2 endpoint."""
# Create a test file
file_path = await create_test_upload_file(tmp_path, chatgpt_json_content)
# Create a multipart form with the file
with open(file_path, "rb") as f:
files = {"file": ("conversations.json", f, "application/json")}
data = {"folder": "test_chatgpt"}
# Send request
response = await client.post(f"{v2_project_url}/import/chatgpt", files=files, data=data)
# Check response
assert response.status_code == 200
result = ChatImportResult.model_validate(response.json())
assert result.success is True
assert result.conversations == 1
assert result.messages == 2
# Verify files were created
conv_path = Path("test_chatgpt") / "20250111-Test_Conversation.md"
assert await file_service.exists(conv_path)
content, _ = await file_service.read_file(conv_path)
assert "# Test Conversation" in content
assert "Hello, this is a test message" in content
assert "This is a test response" in content
@pytest.mark.asyncio
async def test_import_chatgpt_invalid_file(client: AsyncClient, tmp_path, v2_project_url: str):
"""Test importing invalid ChatGPT file via v2 endpoint."""
# Create invalid file
file_path = tmp_path / "invalid.json"
with open(file_path, "w") as f:
f.write("This is not JSON")
# Create multipart form with invalid file
with open(file_path, "rb") as f:
files = {"file": ("invalid.json", f, "application/json")}
data = {"folder": "test_chatgpt"}
# Send request - this should return an error
response = await client.post(f"{v2_project_url}/import/chatgpt", files=files, data=data)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
@pytest.mark.asyncio
async def test_import_claude_conversations(
client: AsyncClient,
tmp_path,
claude_conversations_json_content,
file_service,
v2_project_url: str,
):
"""Test importing Claude conversations via v2 endpoint."""
# Create a test file
file_path = await create_test_upload_file(tmp_path, claude_conversations_json_content)
# Create a multipart form with the file
with open(file_path, "rb") as f:
files = {"file": ("conversations.json", f, "application/json")}
data = {"folder": "test_claude_conversations"}
# Send request
response = await client.post(
f"{v2_project_url}/import/claude/conversations", files=files, data=data
)
# Check response
assert response.status_code == 200
result = ChatImportResult.model_validate(response.json())
assert result.success is True
assert result.conversations == 1
assert result.messages == 2
# Verify files were created
conv_path = Path("test_claude_conversations") / "20250105-Test_Conversation.md"
assert await file_service.exists(conv_path)
content, _ = await file_service.read_file(conv_path)
assert "# Test Conversation" in content
assert "Hello, this is a test" in content
assert "Response to test" in content
@pytest.mark.asyncio
async def test_import_claude_conversations_invalid_file(
client: AsyncClient, tmp_path, v2_project_url: str
):
"""Test importing invalid Claude conversations file via v2 endpoint."""
# Create invalid file
file_path = tmp_path / "invalid.json"
with open(file_path, "w") as f:
f.write("This is not JSON")
# Create multipart form with invalid file
with open(file_path, "rb") as f:
files = {"file": ("invalid.json", f, "application/json")}
data = {"folder": "test_claude_conversations"}
# Send request - this should return an error
response = await client.post(
f"{v2_project_url}/import/claude/conversations", files=files, data=data
)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
@pytest.mark.asyncio
async def test_import_claude_projects(
client: AsyncClient, tmp_path, claude_projects_json_content, file_service, v2_project_url: str
):
"""Test importing Claude projects via v2 endpoint."""
# Create a test file
file_path = await create_test_upload_file(tmp_path, claude_projects_json_content)
# Create a multipart form with the file
with open(file_path, "rb") as f:
files = {"file": ("projects.json", f, "application/json")}
data = {"folder": "test_claude_projects"}
# Send request
response = await client.post(
f"{v2_project_url}/import/claude/projects", files=files, data=data
)
# Check response
assert response.status_code == 200
result = ProjectImportResult.model_validate(response.json())
assert result.success is True
assert result.documents == 2
assert result.prompts == 1
# Verify files were created
project_dir = Path("test_claude_projects") / "Test_Project"
assert await file_service.exists(project_dir / "prompt-template.md")
assert await file_service.exists(project_dir / "docs" / "Test_Document.md")
assert await file_service.exists(project_dir / "docs" / "Another_Document.md")
# Check content
prompt_content, _ = await file_service.read_file(project_dir / "prompt-template.md")
assert "# Test Prompt" in prompt_content
doc_content, _ = await file_service.read_file(project_dir / "docs" / "Test_Document.md")
assert "# Test Document" in doc_content
assert "This is test content" in doc_content
@pytest.mark.asyncio
async def test_import_claude_projects_invalid_file(
client: AsyncClient, tmp_path, v2_project_url: str
):
"""Test importing invalid Claude projects file via v2 endpoint."""
# Create invalid file
file_path = tmp_path / "invalid.json"
with open(file_path, "w") as f:
f.write("This is not JSON")
# Create multipart form with invalid file
with open(file_path, "rb") as f:
files = {"file": ("invalid.json", f, "application/json")}
data = {"folder": "test_claude_projects"}
# Send request - this should return an error
response = await client.post(
f"{v2_project_url}/import/claude/projects", files=files, data=data
)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
@pytest.mark.asyncio
async def test_import_memory_json(
client: AsyncClient, tmp_path, memory_json_content, file_service, v2_project_url: str
):
"""Test importing memory.json file via v2 endpoint."""
# Create a test file
json_file = tmp_path / "memory.json"
with open(json_file, "w", encoding="utf-8") as f:
for entity in memory_json_content:
f.write(json.dumps(entity) + "\n")
# Create a multipart form with the file
with open(json_file, "rb") as f:
files = {"file": ("memory.json", f, "application/json")}
data = {"folder": "test_memory_json"}
# Send request
response = await client.post(f"{v2_project_url}/import/memory-json", files=files, data=data)
# Check response
assert response.status_code == 200
result = EntityImportResult.model_validate(response.json())
assert result.success is True
assert result.entities == 1
assert result.relations == 1
# Verify files were created
entity_path = Path("test_memory_json") / "test" / "test_entity.md"
assert await file_service.exists(entity_path)
# Check content
content, _ = await file_service.read_file(entity_path)
assert "Test observation 1" in content
assert "Test observation 2" in content
assert "test_relation [[related_entity]]" in content
@pytest.mark.asyncio
async def test_import_memory_json_without_folder(
client: AsyncClient, tmp_path, memory_json_content, file_service, v2_project_url: str
):
"""Test importing memory.json file without specifying a destination folder."""
# Create a test file
json_file = tmp_path / "memory.json"
with open(json_file, "w", encoding="utf-8") as f:
for entity in memory_json_content:
f.write(json.dumps(entity) + "\n")
# Create a multipart form with the file
with open(json_file, "rb") as f:
files = {"file": ("memory.json", f, "application/json")}
# Send request without destination_folder
response = await client.post(f"{v2_project_url}/import/memory-json", files=files)
# Check response
assert response.status_code == 200
result = EntityImportResult.model_validate(response.json())
assert result.success is True
assert result.entities == 1
assert result.relations == 1
# Verify files were created in the default directory
entity_path = Path("conversations") / "test" / "test_entity.md"
assert await file_service.exists(entity_path)
@pytest.mark.asyncio
async def test_import_memory_json_invalid_file(client: AsyncClient, tmp_path, v2_project_url: str):
"""Test importing invalid memory.json file via v2 endpoint."""
# Create invalid file
file_path = tmp_path / "invalid.json"
with open(file_path, "w") as f:
f.write("This is not JSON")
# Create multipart form with invalid file
with open(file_path, "rb") as f:
files = {"file": ("invalid.json", f, "application/json")}
data = {"folder": "test_memory_json"}
# Send request - this should return an error
response = await client.post(f"{v2_project_url}/import/memory-json", files=files, data=data)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
@pytest.mark.asyncio
async def test_v2_import_endpoints_use_project_id_not_name(
client: AsyncClient, tmp_path, test_project: Project, chatgpt_json_content
):
"""Verify v2 import endpoints require project ID, not name."""
# Create a test file
file_path = await create_test_upload_file(tmp_path, chatgpt_json_content)
# Try using project name instead of ID - should fail
with open(file_path, "rb") as f:
files = {"file": ("conversations.json", f, "application/json")}
data = {"folder": "test"}
response = await client.post(
f"/v2/projects/{test_project.name}/import/chatgpt",
files=files,
data=data,
)
# Should get validation error or 404 because name is not a valid integer
assert response.status_code in [404, 422]
@pytest.mark.asyncio
async def test_import_invalid_project_id(client: AsyncClient, tmp_path, chatgpt_json_content):
"""Test import endpoints with invalid project ID return 404."""
# Create a test file
file_path = await create_test_upload_file(tmp_path, chatgpt_json_content)
# Test all import endpoints
endpoints = [
"/import/chatgpt",
"/import/claude/conversations",
"/import/claude/projects",
"/import/memory-json",
]
for endpoint in endpoints:
with open(file_path, "rb") as f:
files = {"file": ("test.json", f, "application/json")}
data = {"folder": "test"}
response = await client.post(
f"/v2/projects/999999{endpoint}",
files=files,
data=data,
)
assert response.status_code == 404
@pytest.mark.asyncio
async def test_import_missing_file(client: AsyncClient, v2_project_url: str):
"""Test importing with missing file via v2 endpoint."""
# Send a request without a file
response = await client.post(f"{v2_project_url}/import/chatgpt", data={"folder": "test_folder"})
# Check that the request was rejected
assert response.status_code in [400, 422] # Either bad request or unprocessable entity
@pytest.mark.asyncio
async def test_import_empty_file(client: AsyncClient, tmp_path, v2_project_url: str):
"""Test importing an empty file via v2 endpoint."""
# Create an empty file
file_path = tmp_path / "empty.json"
with open(file_path, "w") as f:
f.write("")
# Create multipart form with empty file
with open(file_path, "rb") as f:
files = {"file": ("empty.json", f, "application/json")}
data = {"folder": "test_chatgpt"}
# Send request
response = await client.post(f"{v2_project_url}/import/chatgpt", files=files, data=data)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
@pytest.mark.asyncio
async def test_import_malformed_json(client: AsyncClient, tmp_path, v2_project_url: str):
"""Test importing malformed JSON for all v2 import endpoints."""
# Create malformed JSON file
file_path = tmp_path / "malformed.json"
with open(file_path, "w") as f:
f.write('{"incomplete": "json"') # Missing closing brace
# Test all import endpoints
endpoints = [
(f"{v2_project_url}/import/chatgpt", {"folder": "test"}),
(f"{v2_project_url}/import/claude/conversations", {"folder": "test"}),
(f"{v2_project_url}/import/claude/projects", {"folder": "test"}),
(f"{v2_project_url}/import/memory-json", {"folder": "test"}),
]
for endpoint, data in endpoints:
# Create multipart form with malformed JSON
with open(file_path, "rb") as f:
files = {"file": ("malformed.json", f, "application/json")}
# Send request
response = await client.post(endpoint, files=files, data=data)
# Check response
assert response.status_code == 500
assert "Import failed" in response.json()["detail"]
-407
View File
@@ -1,407 +0,0 @@
"""Tests for V2 knowledge graph API routes (ID-based endpoints)."""
import pytest
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.schemas import DeleteEntitiesResponse
from basic_memory.schemas.v2 import EntityResponseV2, EntityResolveResponse
@pytest.mark.asyncio
async def test_resolve_identifier_by_permalink(
client: AsyncClient, test_graph, v2_project_url, test_project: Project, entity_repository
):
"""Test resolving an identifier by permalink returns correct entity ID."""
# test_graph fixture creates some test entities
# We'll use one of them to test resolution
# Create an entity first
entity_data = {
"title": "TestResolve",
"folder": "test",
"content": "Test content for resolve",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=entity_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
entity_id = created_entity.id
# Now resolve it by permalink
resolve_data = {"identifier": created_entity.permalink}
response = await client.post(f"{v2_project_url}/knowledge/resolve", json=resolve_data)
assert response.status_code == 200
resolved = EntityResolveResponse.model_validate(response.json())
assert resolved.entity_id == entity_id
assert resolved.permalink == created_entity.permalink
assert resolved.resolution_method == "permalink"
@pytest.mark.asyncio
async def test_resolve_identifier_not_found(client: AsyncClient, v2_project_url):
"""Test resolving a non-existent identifier returns 404."""
resolve_data = {"identifier": "nonexistent/entity"}
response = await client.post(f"{v2_project_url}/knowledge/resolve", json=resolve_data)
assert response.status_code == 404
assert "Could not resolve identifier" in response.json()["detail"]
@pytest.mark.asyncio
async def test_get_entity_by_id(client: AsyncClient, test_graph, v2_project_url, entity_repository):
"""Test getting an entity by its numeric ID."""
# Create an entity first
entity_data = {
"title": "TestGetById",
"folder": "test",
"content": "Test content for get by ID",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=entity_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
entity_id = created_entity.id
# Get it by ID using v2 endpoint
response = await client.get(f"{v2_project_url}/knowledge/entities/{entity_id}")
assert response.status_code == 200
entity = EntityResponseV2.model_validate(response.json())
assert entity.id == entity_id
assert entity.title == "TestGetById"
assert entity.api_version == "v2"
@pytest.mark.asyncio
async def test_get_entity_by_id_not_found(client: AsyncClient, v2_project_url):
"""Test getting a non-existent entity by ID returns 404."""
response = await client.get(f"{v2_project_url}/knowledge/entities/999999")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
@pytest.mark.asyncio
async def test_create_entity(client: AsyncClient, file_service, v2_project_url):
"""Test creating an entity via v2 endpoint."""
data = {
"title": "TestV2Entity",
"folder": "test",
"entity_type": "test",
"content_type": "text/markdown",
"content": "TestContent for V2",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=data)
assert response.status_code == 200
entity = EntityResponseV2.model_validate(response.json())
# V2 endpoints must return id field
assert entity.id is not None
assert isinstance(entity.id, int)
assert entity.api_version == "v2"
assert entity.permalink == "test/test-v2-entity"
assert entity.file_path == "test/TestV2Entity.md"
assert entity.entity_type == data["entity_type"]
# Verify file was created
file_path = file_service.get_entity_path(entity)
file_content, _ = await file_service.read_file(file_path)
assert data["content"] in file_content
@pytest.mark.asyncio
async def test_create_entity_with_observations_and_relations(
client: AsyncClient, file_service, v2_project_url
):
"""Test creating an entity with observations and relations via v2."""
data = {
"title": "TestV2Complex",
"folder": "test",
"content": """
# TestV2Complex
## Observations
- [note] This is a test observation #tag1 (context)
- related to [[OtherEntity]]
""",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=data)
assert response.status_code == 200
entity = EntityResponseV2.model_validate(response.json())
# V2 endpoints must return id field
assert entity.id is not None
assert isinstance(entity.id, int)
assert entity.api_version == "v2"
assert len(entity.observations) == 1
assert entity.observations[0].category == "note"
assert entity.observations[0].content == "This is a test observation #tag1"
assert entity.observations[0].tags == ["tag1"]
assert len(entity.relations) == 1
assert entity.relations[0].relation_type == "related to"
@pytest.mark.asyncio
async def test_update_entity_by_id(
client: AsyncClient, file_service, v2_project_url, entity_repository
):
"""Test updating an entity by ID using PUT (replace)."""
# Create an entity first
create_data = {
"title": "TestUpdate",
"folder": "test",
"content": "Original content",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
original_id = created_entity.id
# Update it by ID
update_data = {
"title": "TestUpdate",
"folder": "test",
"content": "Updated content via V2",
}
response = await client.put(
f"{v2_project_url}/knowledge/entities/{original_id}",
json=update_data,
)
assert response.status_code == 200
updated_entity = EntityResponseV2.model_validate(response.json())
# V2 update must return id field
assert updated_entity.id is not None
assert isinstance(updated_entity.id, int)
assert updated_entity.api_version == "v2"
# Verify file was updated
file_path = file_service.get_entity_path(updated_entity)
file_content, _ = await file_service.read_file(file_path)
assert "Updated content via V2" in file_content
assert "Original content" not in file_content
@pytest.mark.asyncio
async def test_edit_entity_by_id_append(
client: AsyncClient, file_service, v2_project_url, entity_repository
):
"""Test editing an entity by ID using PATCH (append operation)."""
# Create an entity first
create_data = {
"title": "TestEdit",
"folder": "test",
"content": "# TestEdit\n\nOriginal content",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
original_id = created_entity.id
# Edit it by appending
edit_data = {
"operation": "append",
"content": "\n\n## New Section\n\nAppended content",
}
response = await client.patch(
f"{v2_project_url}/knowledge/entities/{original_id}",
json=edit_data,
)
assert response.status_code == 200
edited_entity = EntityResponseV2.model_validate(response.json())
# V2 patch must return id field
assert edited_entity.id is not None
assert isinstance(edited_entity.id, int)
assert edited_entity.api_version == "v2"
# Verify file has both original and appended content
file_path = file_service.get_entity_path(edited_entity)
file_content, _ = await file_service.read_file(file_path)
assert "Original content" in file_content
assert "Appended content" in file_content
@pytest.mark.asyncio
async def test_edit_entity_by_id_find_replace(
client: AsyncClient, file_service, v2_project_url, entity_repository
):
"""Test editing an entity by ID using PATCH (find/replace operation)."""
# Create an entity first
create_data = {
"title": "TestFindReplace",
"folder": "test",
"content": "# TestFindReplace\n\nOld text that will be replaced",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
original_id = created_entity.id
# Edit using find/replace
edit_data = {
"operation": "find_replace",
"find_text": "Old text",
"content": "New text",
}
response = await client.patch(
f"{v2_project_url}/knowledge/entities/{original_id}",
json=edit_data,
)
assert response.status_code == 200
edited_entity = EntityResponseV2.model_validate(response.json())
# V2 patch must return id field
assert edited_entity.id is not None
assert isinstance(edited_entity.id, int)
assert edited_entity.api_version == "v2"
# Verify replacement
file_path = file_service.get_entity_path(created_entity)
file_content, _ = await file_service.read_file(file_path)
assert "New text" in file_content
assert "Old text" not in file_content
@pytest.mark.asyncio
async def test_delete_entity_by_id(
client: AsyncClient, file_service, v2_project_url, entity_repository
):
"""Test deleting an entity by ID."""
# Create an entity first
create_data = {
"title": "TestDelete",
"folder": "test",
"content": "Content to be deleted",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
entity_id = created_entity.id
# Delete it by ID
response = await client.delete(f"{v2_project_url}/knowledge/entities/{entity_id}")
assert response.status_code == 200
delete_response = DeleteEntitiesResponse.model_validate(response.json())
assert delete_response.deleted is True
# Verify it's gone - trying to get it should return 404
response = await client.get(f"{v2_project_url}/knowledge/entities/{entity_id}")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_delete_entity_by_id_not_found(client: AsyncClient, v2_project_url):
"""Test deleting a non-existent entity returns deleted=False (idempotent)."""
response = await client.delete(f"{v2_project_url}/knowledge/entities/999999")
# Delete is idempotent - returns 200 with deleted=False
assert response.status_code == 200
delete_response = DeleteEntitiesResponse.model_validate(response.json())
assert delete_response.deleted is False
@pytest.mark.asyncio
async def test_move_entity(client: AsyncClient, file_service, v2_project_url, entity_repository):
"""Test moving an entity to a new location."""
# Create an entity first
create_data = {
"title": "TestMove",
"folder": "test",
"content": "Content to be moved",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=create_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id
assert created_entity.id is not None
original_id = created_entity.id
# Move it to a new folder (V2 uses entity ID in path)
move_data = {
"destination_path": "moved/MovedEntity.md",
}
response = await client.put(
f"{v2_project_url}/knowledge/entities/{created_entity.id}/move", json=move_data
)
assert response.status_code == 200
moved_entity = EntityResponseV2.model_validate(response.json())
# V2 move must return id field
assert moved_entity.id is not None
assert isinstance(moved_entity.id, int)
assert moved_entity.api_version == "v2"
# ID should remain the same (stable reference)
assert moved_entity.id == original_id
assert moved_entity.file_path == "moved/MovedEntity.md"
@pytest.mark.asyncio
async def test_v2_endpoints_use_project_id_not_name(client: AsyncClient, test_project: Project):
"""Verify v2 endpoints require project ID, not name."""
# Try using project name instead of ID - should fail
response = await client.get(f"/v2/{test_project.name}/knowledge/entities/1")
# Should get validation error or 404 because name is not a valid integer
assert response.status_code in [404, 422]
@pytest.mark.asyncio
async def test_entity_response_v2_has_api_version(
client: AsyncClient, v2_project_url, entity_repository
):
"""Test that EntityResponseV2 includes api_version field."""
# Create an entity
entity_data = {
"title": "TestApiVersion",
"folder": "test",
"content": "Test content",
}
response = await client.post(f"{v2_project_url}/knowledge/entities", json=entity_data)
assert response.status_code == 200
created_entity = EntityResponseV2.model_validate(response.json())
# V2 create must return id and api_version
assert created_entity.id is not None
assert created_entity.api_version == "v2"
entity_id = created_entity.id
# Get it via v2 endpoint
response = await client.get(f"{v2_project_url}/knowledge/entities/{entity_id}")
assert response.status_code == 200
entity_v2 = EntityResponseV2.model_validate(response.json())
assert entity_v2.api_version == "v2"
assert entity_v2.id == entity_id
-301
View File
@@ -1,301 +0,0 @@
"""Tests for v2 memory router endpoints."""
import pytest
from httpx import AsyncClient
from pathlib import Path
from basic_memory.models import Project
async def create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
):
"""Helper to create an entity with file and index it."""
# Create file
test_content = f"# {entity_data['title']}\n\nTest content"
file_path = Path(test_project.path) / entity_data["file_path"]
file_path.parent.mkdir(parents=True, exist_ok=True)
await file_service.write_file(file_path, test_content)
# Create entity
entity = await entity_repository.create(entity_data)
# Index for search
await search_service.index_entity(entity)
return entity
@pytest.mark.asyncio
async def test_get_recent_context(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test getting recent activity context."""
entity_data = {
"title": "Recent Test Entity",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "recent_test.md",
"checksum": "abc123",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get recent context
response = await client.get(f"{v2_project_url}/memory/recent")
assert response.status_code == 200
data = response.json()
# Verify response structure (GraphContext uses 'results' not 'entities')
assert "results" in data
assert "metadata" in data
assert "page" in data
assert "page_size" in data
@pytest.mark.asyncio
async def test_get_recent_context_with_pagination(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test recent context with pagination parameters."""
# Create multiple test entities
for i in range(5):
entity_data = {
"title": f"Entity {i}",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": f"entity_{i}.md",
"checksum": f"checksum{i}",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get recent context with pagination
response = await client.get(
f"{v2_project_url}/memory/recent", params={"page": 1, "page_size": 3}
)
assert response.status_code == 200
data = response.json()
assert "results" in data
assert data["page"] == 1
assert data["page_size"] == 3
@pytest.mark.asyncio
async def test_get_recent_context_with_type_filter(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test filtering recent context by type."""
# Create a test entity
entity_data = {
"title": "Filtered Entity",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "filtered.md",
"checksum": "xyz789",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get recent context filtered by type
response = await client.get(f"{v2_project_url}/memory/recent", params={"type": ["entity"]})
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_get_recent_context_with_timeframe(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test recent context with custom timeframe."""
response = await client.get(f"{v2_project_url}/memory/recent", params={"timeframe": "1d"})
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_get_recent_context_invalid_project_id(
client: AsyncClient,
):
"""Test getting recent context with invalid project ID returns 404."""
response = await client.get("/v2/projects/999999/memory/recent")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_get_memory_context_by_permalink(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test getting context for a specific memory URI (permalink)."""
# Create a test entity
entity_data = {
"title": "Context Test",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "context_test.md",
"checksum": "def456",
"permalink": "context-test",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get context for this entity
response = await client.get(f"{v2_project_url}/memory/context-test")
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_get_memory_context_by_id(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test getting context using ID-based memory URI."""
# Create a test entity
entity_data = {
"title": "ID Context Test",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "id_context_test.md",
"checksum": "ghi789",
}
created_entity = await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get context using ID format (memory://id/123 or memory://123)
response = await client.get(f"{v2_project_url}/memory/id/{created_entity.id}")
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_get_memory_context_with_depth(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test getting context with depth parameter."""
# Create a test entity
entity_data = {
"title": "Depth Test",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "depth_test.md",
"checksum": "jkl012",
"permalink": "depth-test",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get context with depth
response = await client.get(f"{v2_project_url}/memory/depth-test", params={"depth": 2})
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_get_memory_context_not_found(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test getting context for non-existent memory URI returns 404."""
response = await client.get(f"{v2_project_url}/memory/nonexistent-uri")
# Note: This might return 200 with empty results depending on implementation
# Adjust assertion based on actual behavior
assert response.status_code in [200, 404]
@pytest.mark.asyncio
async def test_get_memory_context_with_timeframe(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test getting context with timeframe filter."""
# Create a test entity
entity_data = {
"title": "Timeframe Test",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "timeframe_test.md",
"checksum": "mno345",
"permalink": "timeframe-test",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Get context with timeframe
response = await client.get(
f"{v2_project_url}/memory/timeframe-test", params={"timeframe": "7d"}
)
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_v2_memory_endpoints_use_project_id_not_name(
client: AsyncClient,
test_project: Project,
):
"""Test that v2 memory endpoints reject string project names."""
# Try to use project name instead of ID - should fail
response = await client.get(f"/v2/{test_project.name}/memory/recent")
# FastAPI path validation should reject non-integer project_id
assert response.status_code in [404, 422]
-251
View File
@@ -1,251 +0,0 @@
"""Tests for V2 project management API routes (ID-based endpoints)."""
import tempfile
from pathlib import Path
import pytest
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.schemas.project_info import ProjectItem, ProjectStatusResponse
@pytest.mark.asyncio
async def test_get_project_by_id(client: AsyncClient, test_project: Project, v2_projects_url):
"""Test getting a project by its numeric ID."""
response = await client.get(f"{v2_projects_url}/{test_project.id}")
assert response.status_code == 200
project = ProjectItem.model_validate(response.json())
assert project.id == test_project.id
assert project.name == test_project.name
assert project.path == test_project.path
assert project.is_default == (test_project.is_default or False)
@pytest.mark.asyncio
async def test_get_project_by_id_not_found(client: AsyncClient, v2_projects_url):
"""Test getting a non-existent project by ID returns 404."""
response = await client.get(f"{v2_projects_url}/999999")
assert response.status_code == 404
assert "not found" in response.json()["detail"].lower()
@pytest.mark.asyncio
async def test_update_project_path_by_id(
client: AsyncClient, test_project: Project, v2_projects_url
):
"""Test updating a project's path by ID."""
with tempfile.TemporaryDirectory() as tmpdir:
new_path = str(Path(tmpdir) / "new-project-location")
Path(new_path).mkdir(parents=True, exist_ok=True)
update_data = {"path": new_path}
response = await client.patch(
f"{v2_projects_url}/{test_project.id}",
json=update_data,
)
assert response.status_code == 200
status_response = ProjectStatusResponse.model_validate(response.json())
assert status_response.status == "success"
assert status_response.new_project.id == test_project.id
# Normalize paths for cross-platform comparison (Windows uses backslashes, API returns forward slashes)
assert Path(status_response.new_project.path) == Path(new_path)
assert status_response.old_project.id == test_project.id
@pytest.mark.asyncio
async def test_update_project_invalid_path(
client: AsyncClient, test_project: Project, v2_projects_url
):
"""Test updating with a relative path returns 400."""
update_data = {"path": "relative/path"}
response = await client.patch(
f"{v2_projects_url}/{test_project.id}",
json=update_data,
)
assert response.status_code == 400
assert "absolute" in response.json()["detail"].lower()
@pytest.mark.asyncio
async def test_update_project_not_found(client: AsyncClient, v2_projects_url):
"""Test updating a non-existent project returns 404."""
update_data = {"path": "/tmp/new-path"}
response = await client.patch(
f"{v2_projects_url}/999999",
json=update_data,
)
assert response.status_code == 404
@pytest.mark.asyncio
async def test_set_default_project_by_id(
client: AsyncClient, test_project: Project, v2_projects_url, project_repository, project_service
):
"""Test setting a project as default by ID."""
# Create a second project to test setting default
await project_service.add_project("second-project", "/tmp/second-project")
# Get the created project from the repository to get its ID
created_project = await project_repository.get_by_name("second-project")
assert created_project is not None
# Set the second project as default
response = await client.put(f"{v2_projects_url}/{created_project.id}/default")
assert response.status_code == 200
status_response = ProjectStatusResponse.model_validate(response.json())
assert status_response.status == "success"
assert status_response.default is True
assert status_response.new_project.id == created_project.id
assert status_response.new_project.is_default is True
assert status_response.old_project.id == test_project.id
assert status_response.old_project.is_default is False
@pytest.mark.asyncio
async def test_set_default_project_not_found(client: AsyncClient, v2_projects_url):
"""Test setting a non-existent project as default returns 404."""
response = await client.put(f"{v2_projects_url}/999999/default")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_delete_project_by_id(
client: AsyncClient, test_project: Project, v2_projects_url, project_repository, project_service
):
"""Test deleting a project by ID."""
# Create a second project since we can't delete the default
await project_service.add_project("to-delete", "/tmp/to-delete")
# Get the created project from the repository to get its ID
created_project = await project_repository.get_by_name("to-delete")
assert created_project is not None
# Delete it
response = await client.delete(f"{v2_projects_url}/{created_project.id}")
assert response.status_code == 200
status_response = ProjectStatusResponse.model_validate(response.json())
assert status_response.status == "success"
assert status_response.old_project.id == created_project.id
assert status_response.new_project is None
# Verify it's deleted - trying to get it should return 404
response = await client.get(f"{v2_projects_url}/{created_project.id}")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_delete_project_with_delete_notes_param(
client: AsyncClient, test_project: Project, v2_projects_url, project_repository, project_service
):
"""Test deleting a project with delete_notes parameter."""
# Create a project in a temp directory
with tempfile.TemporaryDirectory() as tmpdir:
project_path = Path(tmpdir) / "test-delete-notes"
project_path.mkdir(parents=True, exist_ok=True)
# Create a test file in the project
test_file = project_path / "test.md"
test_file.write_text("Test content")
await project_service.add_project("delete-with-notes", str(project_path))
# Get the created project from the repository to get its ID
created_project = await project_repository.get_by_name("delete-with-notes")
assert created_project is not None
# Delete with delete_notes=true
response = await client.delete(f"{v2_projects_url}/{created_project.id}?delete_notes=true")
assert response.status_code == 200
# Verify directory was deleted
assert not project_path.exists()
@pytest.mark.asyncio
async def test_delete_default_project_fails(
client: AsyncClient, test_project: Project, v2_projects_url
):
"""Test that deleting the default project returns 400."""
# test_project is the default project
response = await client.delete(f"{v2_projects_url}/{test_project.id}")
assert response.status_code == 400
assert "default project" in response.json()["detail"].lower()
@pytest.mark.asyncio
async def test_delete_project_not_found(client: AsyncClient, v2_projects_url):
"""Test deleting a non-existent project returns 404."""
response = await client.delete(f"{v2_projects_url}/999999")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_v2_project_endpoints_use_id_not_name(
client: AsyncClient, test_project: Project, v2_projects_url
):
"""Verify v2 project endpoints require project ID, not name."""
# Try using project name instead of ID - should fail
response = await client.get(f"{v2_projects_url}/{test_project.name}")
# Should get 404 or 422 because name is not a valid integer
assert response.status_code in [404, 422]
@pytest.mark.asyncio
async def test_project_id_stability_after_rename(
client: AsyncClient, test_project: Project, v2_projects_url, project_repository
):
"""Test that project ID remains stable even after renaming."""
original_id = test_project.id
original_name = test_project.name
# Get project by ID
response = await client.get(f"{v2_projects_url}/{original_id}")
assert response.status_code == 200
project_before = ProjectItem.model_validate(response.json())
assert project_before.id == original_id
assert project_before.name == original_name
# Even if we renamed the project (not testing rename here, just the concept),
# the ID would stay the same. This test demonstrates the stability.
# Re-fetch by same ID
response = await client.get(f"{v2_projects_url}/{original_id}")
assert response.status_code == 200
project_after = ProjectItem.model_validate(response.json())
assert project_after.id == original_id
@pytest.mark.asyncio
async def test_update_project_active_status(
client: AsyncClient, test_project: Project, v2_projects_url, project_repository, project_service
):
"""Test updating a project's active status by ID."""
# Create a non-default project
await project_service.add_project("test-active", "/tmp/test-active")
# Get the created project from the repository to get its ID
created_project = await project_repository.get_by_name("test-active")
assert created_project is not None
# Update active status
update_data = {"is_active": False}
response = await client.patch(
f"{v2_projects_url}/{created_project.id}",
json=update_data,
)
assert response.status_code == 200
status_response = ProjectStatusResponse.model_validate(response.json())
assert status_response.status == "success"
-212
View File
@@ -1,212 +0,0 @@
"""Tests for V2 prompt router endpoints (ID-based)."""
import pytest
import pytest_asyncio
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.services.context_service import ContextService
@pytest_asyncio.fixture
async def context_service(entity_repository, search_service, observation_repository):
"""Create a real context service for testing."""
return ContextService(entity_repository, search_service, observation_repository)
@pytest.mark.asyncio
async def test_continue_conversation_endpoint(
client: AsyncClient,
entity_service,
search_service,
context_service,
entity_repository,
test_graph,
v2_project_url: str,
):
"""Test the v2 continue_conversation endpoint with real services."""
# Create request data
request_data = {
"topic": "Root", # This should match our test entity in test_graph
"timeframe": "7d",
"depth": 1,
"related_items_limit": 2,
}
# Call the endpoint
response = await client.post(
f"{v2_project_url}/prompt/continue-conversation", json=request_data
)
# Verify response
assert response.status_code == 200
result = response.json()
assert "prompt" in result
assert "context" in result
# Check content of context
context = result["context"]
assert context["topic"] == "Root"
assert context["timeframe"] == "7d"
assert context["has_results"] is True
assert len(context["hierarchical_results"]) > 0
# Check content of prompt
prompt = result["prompt"]
assert "Continuing conversation on: Root" in prompt
assert "memory retrieval session" in prompt
@pytest.mark.asyncio
async def test_continue_conversation_without_topic(
client: AsyncClient,
entity_service,
search_service,
context_service,
entity_repository,
test_graph,
v2_project_url: str,
):
"""Test v2 continue_conversation without topic - should use recent activity."""
request_data = {"timeframe": "1d", "depth": 1, "related_items_limit": 2}
response = await client.post(
f"{v2_project_url}/prompt/continue-conversation", json=request_data
)
assert response.status_code == 200
result = response.json()
assert "Recent Activity" in result["context"]["topic"]
@pytest.mark.asyncio
async def test_search_prompt_endpoint(
client: AsyncClient, entity_service, search_service, test_graph, v2_project_url: str
):
"""Test the v2 search_prompt endpoint with real services."""
# Create request data
request_data = {
"query": "Root", # This should match our test entity
"timeframe": "7d",
}
# Call the endpoint
response = await client.post(f"{v2_project_url}/prompt/search", json=request_data)
# Verify response
assert response.status_code == 200
result = response.json()
assert "prompt" in result
assert "context" in result
# Check content of context
context = result["context"]
assert context["query"] == "Root"
assert context["timeframe"] == "7d"
assert context["has_results"] is True
assert len(context["results"]) > 0
# Check content of prompt
prompt = result["prompt"]
assert 'Search Results for: "Root"' in prompt
assert "This is a memory search session" in prompt
@pytest.mark.asyncio
async def test_search_prompt_no_results(
client: AsyncClient, entity_service, search_service, v2_project_url: str
):
"""Test the v2 search_prompt endpoint with a query that returns no results."""
# Create request data with a query that shouldn't match anything
request_data = {"query": "NonExistentQuery12345", "timeframe": "7d"}
# Call the endpoint
response = await client.post(f"{v2_project_url}/prompt/search", json=request_data)
# Verify response
assert response.status_code == 200
result = response.json()
# Check content of context
context = result["context"]
assert context["query"] == "NonExistentQuery12345"
assert context["has_results"] is False
assert len(context["results"]) == 0
# Check content of prompt
prompt = result["prompt"]
assert 'Search Results for: "NonExistentQuery12345"' in prompt
assert "I couldn't find any results for this query" in prompt
assert "Opportunity to Capture Knowledge" in prompt
@pytest.mark.asyncio
async def test_error_handling(client: AsyncClient, monkeypatch, v2_project_url: str):
"""Test error handling in v2 endpoints by breaking the template loader."""
# Patch the template loader to raise an exception
def mock_render(*args, **kwargs):
raise Exception("Template error")
# Apply the patch
monkeypatch.setattr("basic_memory.api.template_loader.TemplateLoader.render", mock_render)
# Test continue_conversation error handling
response = await client.post(
f"{v2_project_url}/prompt/continue-conversation",
json={"topic": "test error", "timeframe": "7d"},
)
assert response.status_code == 500
assert "detail" in response.json()
assert "Template error" in response.json()["detail"]
# Test search_prompt error handling
response = await client.post(
f"{v2_project_url}/prompt/search", json={"query": "test error", "timeframe": "7d"}
)
assert response.status_code == 500
assert "detail" in response.json()
assert "Template error" in response.json()["detail"]
@pytest.mark.asyncio
async def test_v2_prompt_endpoints_use_project_id_not_name(
client: AsyncClient, test_project: Project
):
"""Verify v2 prompt endpoints require project ID, not name."""
# Try using project name instead of ID - should fail
response = await client.post(
f"/v2/projects/{test_project.name}/prompt/continue-conversation",
json={"topic": "test", "timeframe": "7d"},
)
# Should get validation error or 404 because name is not a valid integer
assert response.status_code in [404, 422]
# Also test search endpoint
response = await client.post(
f"/v2/projects/{test_project.name}/prompt/search",
json={"query": "test", "timeframe": "7d"},
)
assert response.status_code in [404, 422]
@pytest.mark.asyncio
async def test_prompt_invalid_project_id(client: AsyncClient):
"""Test prompt endpoints with invalid project ID return 404."""
# Test continue-conversation
response = await client.post(
"/v2/projects/999999/prompt/continue-conversation",
json={"topic": "test", "timeframe": "7d"},
)
assert response.status_code == 404
# Test search
response = await client.post(
"/v2/projects/999999/prompt/search",
json={"query": "test", "timeframe": "7d"},
)
assert response.status_code == 404
-267
View File
@@ -1,267 +0,0 @@
"""Tests for V2 resource API routes (ID-based endpoints)."""
import pytest
from httpx import AsyncClient
from basic_memory.models import Project
from basic_memory.schemas.v2.resource import ResourceResponse
@pytest.mark.asyncio
async def test_create_resource(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test creating a new resource via v2 POST endpoint."""
create_data = {
"file_path": "test-resources/test-file.md",
"content": "# Test Resource\n\nThis is test content.",
}
response = await client.post(
f"{v2_project_url}/resource",
json=create_data,
)
assert response.status_code == 200
result = ResourceResponse.model_validate(response.json())
# V2 must return entity_id
assert result.entity_id is not None
assert isinstance(result.entity_id, int)
assert result.file_path == "test-resources/test-file.md"
assert result.checksum is not None
@pytest.mark.asyncio
async def test_create_resource_duplicate_fails(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test that creating a resource at an existing path returns 409."""
create_data = {
"file_path": "duplicate-test.md",
"content": "First version",
}
# Create first time - should succeed
response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert response.status_code == 200
# Try to create again - should fail with 409
response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert response.status_code == 409
assert "already exists" in response.json()["detail"]
@pytest.mark.asyncio
async def test_get_resource_by_id(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test getting resource content by entity ID."""
# First create a resource
test_content = "# Test Resource\n\nThis is test content."
create_data = {
"file_path": "test-get.md",
"content": test_content,
}
create_response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert create_response.status_code == 200
created = ResourceResponse.model_validate(create_response.json())
# Now get it by entity ID
response = await client.get(f"{v2_project_url}/resource/{created.entity_id}")
assert response.status_code == 200
# Normalize line endings for cross-platform compatibility
assert test_content.replace("\n", "") in response.text.replace("\r\n", "").replace("\n", "")
@pytest.mark.asyncio
async def test_get_resource_not_found(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test getting a non-existent resource returns 404."""
response = await client.get(f"{v2_project_url}/resource/999999")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_update_resource(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test updating resource content by entity ID."""
# Create a resource
create_data = {
"file_path": "test-update.md",
"content": "Original content",
}
create_response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert create_response.status_code == 200
created = ResourceResponse.model_validate(create_response.json())
# Update it
update_data = {
"content": "Updated content",
}
response = await client.put(
f"{v2_project_url}/resource/{created.entity_id}",
json=update_data,
)
assert response.status_code == 200
result = ResourceResponse.model_validate(response.json())
assert result.entity_id == created.entity_id
assert result.file_path == "test-update.md"
# Verify content was updated
get_response = await client.get(f"{v2_project_url}/resource/{created.entity_id}")
assert "Updated content" in get_response.text
@pytest.mark.asyncio
async def test_update_resource_and_move(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test updating resource content and moving it to a new path."""
# Create a resource
create_data = {
"file_path": "original-location.md",
"content": "Original content",
}
create_response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert create_response.status_code == 200
created = ResourceResponse.model_validate(create_response.json())
# Update content and move file
update_data = {
"content": "Updated content in new location",
"file_path": "moved/new-location.md",
}
response = await client.put(
f"{v2_project_url}/resource/{created.entity_id}",
json=update_data,
)
assert response.status_code == 200
result = ResourceResponse.model_validate(response.json())
assert result.entity_id == created.entity_id
assert result.file_path == "moved/new-location.md"
# Verify content at new location
get_response = await client.get(f"{v2_project_url}/resource/{created.entity_id}")
assert "Updated content in new location" in get_response.text
@pytest.mark.asyncio
async def test_update_resource_not_found(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test updating a non-existent resource returns 404."""
update_data = {
"content": "New content",
}
response = await client.put(
f"{v2_project_url}/resource/999999",
json=update_data,
)
assert response.status_code == 404
@pytest.mark.asyncio
async def test_create_resource_invalid_path(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test creating a resource with path traversal attempt fails."""
create_data = {
"file_path": "../../../etc/passwd",
"content": "malicious content",
}
response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert response.status_code == 400
assert "Invalid file path" in response.json()["detail"]
@pytest.mark.asyncio
async def test_update_resource_invalid_path(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test updating a resource with path traversal attempt fails."""
# Create a valid resource first
create_data = {
"file_path": "valid.md",
"content": "Valid content",
}
create_response = await client.post(f"{v2_project_url}/resource", json=create_data)
assert create_response.status_code == 200
created = ResourceResponse.model_validate(create_response.json())
# Try to move it to an invalid path
update_data = {
"content": "Updated content",
"file_path": "../../../etc/passwd",
}
response = await client.put(
f"{v2_project_url}/resource/{created.entity_id}",
json=update_data,
)
assert response.status_code == 400
assert "Invalid file path" in response.json()["detail"]
@pytest.mark.asyncio
async def test_resource_invalid_project_id(
client: AsyncClient,
):
"""Test resource endpoints with invalid project ID return 404."""
# Test create
response = await client.post(
"/v2/projects/999999/resource",
json={"file_path": "test.md", "content": "test"},
)
assert response.status_code == 404
# Test get
response = await client.get("/v2/projects/999999/resource/1")
assert response.status_code == 404
# Test update
response = await client.put(
"/v2/projects/999999/resource/1",
json={"content": "test"},
)
assert response.status_code == 404
@pytest.mark.asyncio
async def test_v2_resource_endpoints_use_project_id_not_name(
client: AsyncClient, test_project: Project
):
"""Verify v2 resource endpoints require project ID, not name."""
# Try using project name instead of ID - should fail
response = await client.get(f"/v2/projects/{test_project.name}/resource/1")
# Should get validation error or 404 because name is not a valid integer
assert response.status_code in [404, 422]
-289
View File
@@ -1,289 +0,0 @@
"""Tests for v2 search router endpoints."""
import pytest
from httpx import AsyncClient
from pathlib import Path
from basic_memory.models import Project
async def create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
):
"""Helper to create an entity with file and index it."""
# Create file
test_content = f"# {entity_data['title']}\n\nTest content"
file_path = Path(test_project.path) / entity_data["file_path"]
file_path.parent.mkdir(parents=True, exist_ok=True)
await file_service.write_file(file_path, test_content)
# Create entity
entity = await entity_repository.create(entity_data)
# Index for search
await search_service.index_entity(entity)
return entity
@pytest.mark.asyncio
async def test_search_entities(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test searching for entities."""
# Create a test entity
entity_data = {
"title": "Searchable Entity",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "searchable.md",
"checksum": "search123",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search for the entity
response = await client.post(f"{v2_project_url}/search/", json={"search_text": "Searchable"})
assert response.status_code == 200
data = response.json()
# Verify response structure
assert "results" in data
assert "current_page" in data
assert "page_size" in data
@pytest.mark.asyncio
async def test_search_with_pagination(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test search with pagination parameters."""
# Create multiple test entities
for i in range(5):
entity_data = {
"title": f"Search Entity {i}",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": f"search_{i}.md",
"checksum": f"searchsum{i}",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search with pagination
response = await client.post(
f"{v2_project_url}/search/",
json={"search_text": "Search Entity"},
params={"page": 1, "page_size": 3},
)
assert response.status_code == 200
data = response.json()
assert data["current_page"] == 1
assert data["page_size"] == 3
@pytest.mark.asyncio
async def test_search_by_permalink(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test searching by permalink."""
# Create a test entity with permalink
entity_data = {
"title": "Permalink Search",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "permalink_search.md",
"checksum": "perm123",
"permalink": "permalink-search",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search by permalink
response = await client.post(
f"{v2_project_url}/search/", json={"permalink": "permalink-search"}
)
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_search_by_title(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test searching by title."""
# Create a test entity
entity_data = {
"title": "Unique Title For Search",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "unique_title.md",
"checksum": "title123",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search by title
response = await client.post(f"{v2_project_url}/search/", json={"title": "Unique Title"})
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_search_with_type_filter(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test searching with entity type filter."""
# Create test entities of different types
for entity_type in ["note", "document"]:
entity_data = {
"title": f"Type {entity_type}",
"entity_type": entity_type,
"content_type": "text/markdown",
"file_path": f"type_{entity_type}.md",
"checksum": f"type{entity_type}",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search with type filter
response = await client.post(
f"{v2_project_url}/search/", json={"search_text": "Type", "types": ["note"]}
)
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_search_with_date_filter(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
entity_repository,
search_service,
file_service,
):
"""Test searching with date filter."""
# Create a test entity
entity_data = {
"title": "Date Filtered",
"entity_type": "note",
"content_type": "text/markdown",
"file_path": "date_filtered.md",
"checksum": "date123",
}
await create_test_entity(
test_project, entity_data, entity_repository, search_service, file_service
)
# Search with date filter
response = await client.post(
f"{v2_project_url}/search/",
json={"search_text": "Date Filtered", "after_date": "2024-01-01T00:00:00Z"},
)
assert response.status_code == 200
data = response.json()
assert "results" in data
@pytest.mark.asyncio
async def test_search_empty_query(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test search with empty query."""
response = await client.post(f"{v2_project_url}/search/", json={})
# Empty query should still be valid (returns all)
assert response.status_code in [200, 422]
@pytest.mark.asyncio
async def test_search_invalid_project_id(
client: AsyncClient,
):
"""Test searching with invalid project ID returns 404."""
response = await client.post("/v2/projects/999999/search/", json={"search_text": "test"})
assert response.status_code == 404
@pytest.mark.asyncio
async def test_reindex(
client: AsyncClient,
test_project: Project,
v2_project_url: str,
):
"""Test reindexing search index."""
response = await client.post(f"{v2_project_url}/search/reindex")
assert response.status_code == 200
data = response.json()
# Verify response structure
assert "status" in data
assert data["status"] == "ok"
assert "message" in data
@pytest.mark.asyncio
async def test_reindex_invalid_project_id(
client: AsyncClient,
):
"""Test reindexing with invalid project ID returns 404."""
response = await client.post("/v2/projects/999999/search/reindex")
assert response.status_code == 404
@pytest.mark.asyncio
async def test_v2_search_endpoints_use_project_id_not_name(
client: AsyncClient,
test_project: Project,
):
"""Test that v2 search endpoints reject string project names."""
# Try to use project name instead of ID - should fail
response = await client.post(f"/v2/{test_project.name}/search/", json={"search_text": "test"})
# FastAPI path validation should reject non-integer project_id
assert response.status_code in [404, 422]
@@ -55,7 +55,6 @@ def mock_api_client():
"default": False,
"old_project": None,
"new_project": {
"id": 1,
"name": "test-project",
"path": "/test-project",
"is_default": False,
-29
View File
@@ -121,35 +121,6 @@ def test_observation_excludes_markdown_and_wiki_links():
assert not is_observation(token), "No space after category should not be valid observation"
def test_observation_html_color_codes():
"""Test that HTML color codes are NOT parsed as hashtags (issue #446).
This validates the fix where content like:
- **<font color="#4285F4">Jane:</font>** Welcome...
should NOT be treated as an observation due to the HTML color code.
"""
# Test HTML color codes should NOT be detected as hashtags
token = Token("inline", '**<font color="#4285F4">Jane:</font>** Welcome to the deep dive', 0)
assert not is_observation(token), "HTML color codes should not be parsed as hashtags"
token = Token("inline", '<font color="#FF0000">Red text</font>', 0)
assert not is_observation(token), "HTML color codes should not be parsed as hashtags"
token = Token("inline", 'Style: background-color=#ABCDEF', 0)
assert not is_observation(token), "CSS color codes should not be parsed as hashtags"
# Test valid hashtags still work
token = Token("inline", "This is a note #hashtag", 0)
assert is_observation(token), "Valid hashtags should still be detected"
token = Token("inline", "[note] Content with #tag1 and #tag2", 0)
assert is_observation(token), "Valid hashtags in observations should still work"
# Test edge case: hashtag next to hex but separated by space
token = Token("inline", "Color is ABCDEF #hashtag", 0)
assert is_observation(token), "Hashtags after hex should work if properly separated"
def test_relation_plugin():
"""Test relation plugin."""
md = MarkdownIt().use(relation_plugin)
-1
View File
@@ -116,7 +116,6 @@ def test_prompt_context_with_file_path_no_permalink():
# Create a mock context with a file that has no permalink (like a binary file)
test_entity = EntitySummary(
entity_id=1,
type="entity",
title="Test File",
permalink=None, # No permalink
@@ -356,62 +356,3 @@ async def test_find_by_category_case_sensitivity(
upper_case = await repo.find_by_category("TECH")
assert len(upper_case) == 0 # Currently case-sensitive
@pytest.mark.asyncio
async def test_observation_permalink_truncation(
session_maker: async_sessionmaker, repo, test_project: Project
):
"""Test that observation permalinks are truncated to prevent database index overflow (issue #446).
PostgreSQL btree indexes have a maximum size of 2704 bytes. Long observation content
was causing permalinks to exceed this limit. The fix truncates content to 150 chars
when generating permalinks.
"""
async with db.scoped_session(session_maker) as session:
entity = Entity(
project_id=test_project.id,
title="test_entity",
entity_type="test",
permalink="test/test-entity",
file_path="test/test_entity.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
)
session.add(entity)
await session.flush()
# Create observation with very long content (simulating transcript dialogue)
long_content = "This is a very long observation content that would normally cause the permalink to exceed the PostgreSQL btree index limit of 2704 bytes. " * 50
obs = Observation(
entity_id=entity.id,
content=long_content,
category="transcript",
)
session.add(obs)
await session.commit()
# Refresh to get the relationship
await session.refresh(obs)
await session.refresh(obs, ["entity"])
# Verify the full content is stored
assert len(obs.content) > 150
assert obs.content == long_content
# Verify the permalink is truncated
permalink = obs.permalink
assert permalink is not None
assert len(permalink) < 500 # Should be much shorter than full content
# Verify permalink contains truncated content, not full content
# The permalink format is: {entity.permalink}/observations/{category}/{truncated_content}
assert "test-entity/observations/transcript/" in permalink
# Verify the content portion is limited
# Extract the content portion after the last slash
content_portion = permalink.split("/")[-1]
# The content portion should correspond to truncated content (150 chars max)
# After generate_permalink processing (lowercase, spaces to hyphens, etc.)
assert len(content_portion) <= 200 # Allow some buffer for URL encoding
+1 -20
View File
@@ -22,7 +22,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 10, 30, 0)
entity = EntitySummary(
entity_id=1,
permalink="test/entity",
title="Test Entity",
file_path="test/entity.md",
@@ -42,8 +41,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 15, 45, 30)
relation = RelationSummary(
relation_id=1,
entity_id=1,
title="Test Relation",
file_path="test/relation.md",
permalink="test/relation",
@@ -66,8 +63,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 20, 15, 45)
observation = ObservationSummary(
observation_id=1,
entity_id=1,
title="Test Observation",
file_path="test/observation.md",
permalink="test/observation",
@@ -105,7 +100,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 9, 30, 15)
entity = EntitySummary(
entity_id=1,
permalink="test/entity",
title="Test Entity",
file_path="test/entity.md",
@@ -113,8 +107,6 @@ class TestDateTimeSerialization:
)
observation = ObservationSummary(
observation_id=1,
entity_id=1,
title="Test Observation",
file_path="test/observation.md",
permalink="test/observation",
@@ -139,7 +131,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 14, 20, 10)
entity = EntitySummary(
entity_id=1,
permalink="test/entity",
title="Test Entity",
file_path="test/entity.md",
@@ -168,7 +159,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 10, 30, 0, 123456)
entity = EntitySummary(
entity_id=1,
permalink="test/entity",
title="Test Entity",
file_path="test/entity.md",
@@ -186,7 +176,6 @@ class TestDateTimeSerialization:
test_datetime = datetime(2023, 12, 8, 10, 30, 0)
entity = EntitySummary(
entity_id=1,
permalink="test/entity",
title="Test Entity",
file_path="test/entity.md",
@@ -223,16 +212,10 @@ class TestDateTimeSerialization:
if model_class == EntitySummary:
instance = model_class(
entity_id=1,
permalink="test",
title="Test",
file_path="test.md",
created_at=test_datetime,
permalink="test", title="Test", file_path="test.md", created_at=test_datetime
)
elif model_class == RelationSummary:
instance = model_class(
relation_id=1,
entity_id=1,
title="Test",
file_path="test.md",
permalink="test",
@@ -241,8 +224,6 @@ class TestDateTimeSerialization:
)
elif model_class == ObservationSummary:
instance = model_class(
observation_id=1,
entity_id=1,
title="Test",
file_path="test.md",
permalink="test",
-179
View File
@@ -755,182 +755,3 @@ async def test_search_title_via_repository_direct(search_service, session_maker,
# Should find the entity without throwing FTS5 syntax errors
assert len(results) >= 1
assert any(result.title == "Note (with parentheses)" for result in results)
# Tests for duplicate observation permalink deduplication
@pytest.mark.asyncio
async def test_index_entity_with_duplicate_observations(
search_service, session_maker, test_project
):
"""Test that indexing an entity with duplicate observations doesn't cause unique constraint violations.
Two observations with the same category and content generate identical permalinks,
which would violate the unique constraint on the search_index table.
"""
from basic_memory.repository import EntityRepository, ObservationRepository
from unittest.mock import AsyncMock
from datetime import datetime
entity_repo = EntityRepository(session_maker, project_id=test_project.id)
obs_repo = ObservationRepository(session_maker, project_id=test_project.id)
# Create entity
entity_data = {
"title": "Entity With Duplicate Observations",
"entity_type": "note",
"entity_metadata": {},
"content_type": "text/markdown",
"file_path": "test/duplicate-obs.md",
"permalink": "test/duplicate-obs",
"project_id": test_project.id,
"created_at": datetime.now(),
"updated_at": datetime.now(),
}
entity = await entity_repo.create(entity_data)
# Create duplicate observations - same category and content
duplicate_content = "This is a duplicated observation"
await obs_repo.create(
{"entity_id": entity.id, "category": "note", "content": duplicate_content}
)
await obs_repo.create(
{"entity_id": entity.id, "category": "note", "content": duplicate_content}
)
# Reload entity with observations (get_by_permalink eagerly loads observations)
entity = await entity_repo.get_by_permalink("test/duplicate-obs")
# Verify we have duplicate observations
assert len(entity.observations) == 2
assert entity.observations[0].permalink == entity.observations[1].permalink
# Mock file service to avoid file I/O
search_service.file_service.read_entity_content = AsyncMock(return_value="")
# This should not raise a unique constraint violation
await search_service.index_entity(entity)
# Verify entity is searchable
results = await search_service.search(SearchQuery(text="Duplicate Observations"))
assert len(results) >= 1
assert any(r.title == "Entity With Duplicate Observations" for r in results)
@pytest.mark.asyncio
async def test_index_entity_dedupes_observations_by_permalink(
search_service, session_maker, test_project
):
"""Test that only unique observation permalinks are indexed.
When an entity has observations with identical permalinks, only the first one
should be indexed to avoid unique constraint violations.
"""
from basic_memory.repository import EntityRepository, ObservationRepository
from unittest.mock import AsyncMock
from datetime import datetime
entity_repo = EntityRepository(session_maker, project_id=test_project.id)
obs_repo = ObservationRepository(session_maker, project_id=test_project.id)
# Create entity
entity_data = {
"title": "Dedupe Test Entity",
"entity_type": "note",
"entity_metadata": {},
"content_type": "text/markdown",
"file_path": "test/dedupe-test.md",
"permalink": "test/dedupe-test",
"project_id": test_project.id,
"created_at": datetime.now(),
"updated_at": datetime.now(),
}
entity = await entity_repo.create(entity_data)
# Create three observations: two duplicates and one unique
duplicate_content = "Duplicate observation content"
unique_content = "Unique observation content"
await obs_repo.create(
{"entity_id": entity.id, "category": "note", "content": duplicate_content}
)
await obs_repo.create(
{"entity_id": entity.id, "category": "note", "content": duplicate_content}
)
await obs_repo.create({"entity_id": entity.id, "category": "note", "content": unique_content})
# Reload entity with observations (get_by_permalink eagerly loads observations)
entity = await entity_repo.get_by_permalink("test/dedupe-test")
assert len(entity.observations) == 3
# Mock file service to avoid file I/O
search_service.file_service.read_entity_content = AsyncMock(return_value="")
# Index the entity
await search_service.index_entity(entity)
# Search for the unique observation - should find it
results = await search_service.search(SearchQuery(text="Unique observation"))
assert len(results) >= 1
# Search for duplicate observation - should find it (only one indexed)
results = await search_service.search(SearchQuery(text="Duplicate observation"))
assert len(results) >= 1
@pytest.mark.asyncio
async def test_index_entity_multiple_categories_same_content(
search_service, session_maker, test_project
):
"""Test that observations with same content but different categories are not deduped.
The permalink includes the category, so observations with different categories
but same content should have different permalinks and both be indexed.
"""
from basic_memory.repository import EntityRepository, ObservationRepository
from unittest.mock import AsyncMock
from datetime import datetime
entity_repo = EntityRepository(session_maker, project_id=test_project.id)
obs_repo = ObservationRepository(session_maker, project_id=test_project.id)
# Create entity
entity_data = {
"title": "Multi Category Entity",
"entity_type": "note",
"entity_metadata": {},
"content_type": "text/markdown",
"file_path": "test/multi-category.md",
"permalink": "test/multi-category",
"project_id": test_project.id,
"created_at": datetime.now(),
"updated_at": datetime.now(),
}
entity = await entity_repo.create(entity_data)
# Create observations with same content but different categories
shared_content = "Shared content across categories"
await obs_repo.create({"entity_id": entity.id, "category": "tech", "content": shared_content})
await obs_repo.create({"entity_id": entity.id, "category": "design", "content": shared_content})
# Reload entity with observations (get_by_permalink eagerly loads observations)
entity = await entity_repo.get_by_permalink("test/multi-category")
assert len(entity.observations) == 2
# Verify permalinks are different due to different categories
permalinks = {obs.permalink for obs in entity.observations}
assert len(permalinks) == 2 # Should be 2 unique permalinks
# Mock file service to avoid file I/O
search_service.file_service.read_entity_content = AsyncMock(return_value="")
# Index the entity - both should be indexed since permalinks differ
await search_service.index_entity(entity)
# Search for the shared content - should find both observations
results = await search_service.search(SearchQuery(text="Shared content"))
assert len(results) >= 2
+16 -98
View File
@@ -106,89 +106,6 @@ Target content
assert source.relations[0].to_name == target.title
@pytest.mark.asyncio
async def test_resolve_relations_deletes_duplicate_unresolved_relation(
sync_service: SyncService,
project_config: ProjectConfig,
entity_service: EntityService,
):
"""Test that resolve_relations deletes duplicate unresolved relations on IntegrityError.
When resolving a forward reference would create a duplicate (from_id, to_id, relation_type),
the unresolved relation should be deleted since a resolved version already exists.
"""
from unittest.mock import patch
from sqlalchemy.exc import IntegrityError
from basic_memory.models import Relation
project_dir = project_config.home
# Create source entity
source_content = """
---
type: knowledge
---
# Source Entity
Content
"""
await create_test_file(project_dir / "source.md", source_content)
# Create target entity
target_content = """
---
type: knowledge
---
# Target Entity
Content
"""
await create_test_file(project_dir / "target.md", target_content)
# Sync to create both entities
await sync_service.sync(project_config.home)
source = await entity_service.get_by_permalink("source")
await entity_service.get_by_permalink("target")
# Create an unresolved relation that will resolve to target
unresolved_relation = Relation(
from_id=source.id,
to_id=None, # Unresolved
to_name="target", # Will resolve to target entity
relation_type="relates_to",
)
await sync_service.relation_repository.add(unresolved_relation)
unresolved_id = unresolved_relation.id
# Verify we have the unresolved relation
source = await entity_service.get_by_permalink("source")
assert len(source.outgoing_relations) == 1
assert source.outgoing_relations[0].to_id is None
# Mock the repository update to raise IntegrityError (simulating existing duplicate)
async def mock_update_raises_integrity_error(entity_id, data):
# Simulate: a resolved relation with same (from_id, to_id, relation_type) already exists
raise IntegrityError(
"UNIQUE constraint failed: relation.from_id, relation.to_id, relation.relation_type",
None,
None,
)
with patch.object(
sync_service.relation_repository, "update", side_effect=mock_update_raises_integrity_error
):
# Call resolve_relations - should hit IntegrityError and delete the duplicate
await sync_service.resolve_relations()
# Verify the unresolved relation was deleted
deleted = await sync_service.relation_repository.find_by_id(unresolved_id)
assert deleted is None
# Verify no unresolved relations remain
unresolved = await sync_service.relation_repository.find_unresolved_relations()
assert len(unresolved) == 0
@pytest.mark.asyncio
async def test_sync(
sync_service: SyncService, project_config: ProjectConfig, entity_service: EntityService
@@ -1569,11 +1486,12 @@ async def test_circuit_breaker_skips_after_three_failures(
# Create a file with malformed content that will fail to parse
await create_test_file(test_file, "invalid markdown content")
# Mock sync_markdown_file to always fail
async def mock_sync_markdown_file(*args, **kwargs):
# Mock sync_markdown_batch to always fail for all files in batch
async def mock_sync_markdown_batch(paths, new=True):
# Simulate batch failure - return (None, "") for each path
raise ValueError("Simulated sync failure")
with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file):
with patch.object(sync_service, "sync_markdown_batch", side_effect=mock_sync_markdown_batch):
# First sync - should fail and record (1/3)
report1 = await sync_service.sync(project_dir)
assert len(report1.skipped_files) == 0 # Not skipped yet
@@ -1628,15 +1546,15 @@ async def test_circuit_breaker_resets_on_file_change(
# Create initial failing content
await create_test_file(test_file, "initial bad content")
# Mock sync_markdown_file to fail
# Mock sync_markdown_batch to fail
call_count = 0
async def mock_sync_markdown_file(*args, **kwargs):
async def mock_sync_markdown_batch(paths, new=True):
nonlocal call_count
call_count += 1
call_count += len(paths)
raise ValueError("Simulated sync failure")
with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file):
with patch.object(sync_service, "sync_markdown_batch", side_effect=mock_sync_markdown_batch):
# Fail 3 times to hit circuit breaker threshold
await sync_service.sync(project_dir) # Fail 1
await touch_file(test_file) # Touch to trigger incremental scan
@@ -1753,8 +1671,8 @@ async def test_circuit_breaker_handles_checksum_computation_failure(
test_file = project_dir / "checksum_fail.md"
await create_test_file(test_file, "content")
# Mock sync_markdown_file to fail
async def mock_sync_markdown_file(*args, **kwargs):
# Mock sync_markdown_batch to fail
async def mock_sync_markdown_batch(paths, new=True):
raise ValueError("Sync failure")
# Mock checksum computation to fail only during _record_failure (not during scan)
@@ -1771,7 +1689,7 @@ async def test_circuit_breaker_handles_checksum_computation_failure(
raise IOError("Cannot read file")
with (
patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file),
patch.object(sync_service, "sync_markdown_batch", side_effect=mock_sync_markdown_batch),
patch.object(
sync_service.file_service,
"compute_checksum",
@@ -1840,16 +1758,16 @@ async def test_sync_fatal_error_terminates_sync_immediately(
),
)
# Mock entity_service.create_entity_from_markdown to raise SyncFatalError on first file
# This simulates project being deleted during sync
async def mock_create_entity_from_markdown(*args, **kwargs):
# Mock entity_repository.upsert_entities to raise SyncFatalError
# This simulates project being deleted during batch sync
async def mock_upsert_entities(entities):
raise SyncFatalError(
"Cannot sync file 'file1.md': project_id=99999 does not exist in database. "
"Cannot sync entities: project_id=99999 does not exist in database. "
"The project may have been deleted. This sync will be terminated."
)
with patch.object(
entity_service, "create_entity_from_markdown", side_effect=mock_create_entity_from_markdown
sync_service.entity_repository, "upsert_entities", side_effect=mock_upsert_entities
):
# Sync should raise SyncFatalError and terminate immediately
with pytest.raises(SyncFatalError, match="project_id=99999 does not exist"):
Generated
+49 -59
View File
@@ -60,15 +60,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" },
]
[[package]]
name = "asgiref"
version = "3.11.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/76/b9/4db2509eabd14b4a8c71d1b24c8d5734c52b8560a7b1e1a8b56c8d25568b/asgiref-3.11.0.tar.gz", hash = "sha256:13acff32519542a1736223fb79a715acdebe24286d98e8b164a73085f40da2c4", size = 37969, upload-time = "2025-11-19T15:32:20.106Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/91/be/317c2c55b8bbec407257d45f5c8d1b6867abc76d12043f2d3d58c538a4ea/asgiref-3.11.0-py3-none-any.whl", hash = "sha256:1db9021efadb0d9512ce8ffaf72fcef601c7b73a8807a1bb2ef143dc6b14846d", size = 24096, upload-time = "2025-11-19T15:32:19.004Z" },
]
[[package]]
name = "asttokens"
version = "3.0.0"
@@ -135,11 +126,11 @@ dependencies = [
{ name = "fastapi", extra = ["standard"] },
{ name = "fastmcp" },
{ name = "greenlet" },
{ name = "logfire", extra = ["fastapi"] },
{ name = "icecream" },
{ name = "logfire" },
{ name = "loguru" },
{ name = "markdown-it-py" },
{ name = "mcp" },
{ name = "nest-asyncio" },
{ name = "pillow" },
{ name = "pybars3" },
{ name = "pydantic", extra = ["email", "timezone"] },
@@ -162,6 +153,8 @@ dev = [
{ name = "freezegun" },
{ name = "gevent" },
{ name = "icecream" },
{ name = "nest-asyncio" },
{ name = "psycopg2-binary" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
@@ -180,11 +173,11 @@ requires-dist = [
{ name = "fastapi", extras = ["standard"], specifier = ">=0.115.8" },
{ name = "fastmcp", specifier = ">=2.10.2" },
{ name = "greenlet", specifier = ">=3.1.1" },
{ name = "logfire", extras = ["fastapi"], specifier = ">=0.73.0" },
{ name = "icecream", specifier = ">=2.1.3" },
{ name = "logfire", specifier = ">=0.73.0" },
{ name = "loguru", specifier = ">=0.7.3" },
{ name = "markdown-it-py", specifier = ">=3.0.0" },
{ name = "mcp", specifier = ">=1.2.0" },
{ name = "nest-asyncio", specifier = ">=1.6.0" },
{ name = "pillow", specifier = ">=11.1.0" },
{ name = "pybars3", specifier = ">=0.9.7" },
{ name = "pydantic", extras = ["email", "timezone"], specifier = ">=2.10.3" },
@@ -207,6 +200,8 @@ dev = [
{ name = "freezegun", specifier = ">=1.5.5" },
{ name = "gevent", specifier = ">=24.11.1" },
{ name = "icecream", specifier = ">=2.1.3" },
{ name = "nest-asyncio", specifier = ">=1.6.0" },
{ name = "psycopg2-binary", specifier = ">=2.9.0" },
{ name = "pytest", specifier = ">=8.3.4" },
{ name = "pytest-asyncio", specifier = ">=0.24.0" },
{ name = "pytest-cov", specifier = ">=4.1.0" },
@@ -906,11 +901,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4b/5f/0803848cc5ce524ff830e5f2a2f1400fd6ee72be705d87d0432cec42b1e4/logfire-4.13.2-py3-none-any.whl", hash = "sha256:887e99897a1818864aa5bfc595b02c93264ce23d1860866369eff6b6e2dde1c6", size = 228152, upload-time = "2025-10-13T16:17:50.641Z" },
]
[package.optional-dependencies]
fastapi = [
{ name = "opentelemetry-instrumentation-fastapi" },
]
[[package]]
name = "loguru"
version = "0.7.3"
@@ -1163,38 +1153,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d4/db/5ff1cd6c5ca1d12ecf1b73be16fbb2a8af2114ee46d4b0e6d4b23f4f4db7/opentelemetry_instrumentation-0.58b0-py3-none-any.whl", hash = "sha256:50f97ac03100676c9f7fc28197f8240c7290ca1baa12da8bfbb9a1de4f34cc45", size = 33019, upload-time = "2025-09-11T11:41:00.624Z" },
]
[[package]]
name = "opentelemetry-instrumentation-asgi"
version = "0.58b0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "asgiref" },
{ name = "opentelemetry-api" },
{ name = "opentelemetry-instrumentation" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "opentelemetry-util-http" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7b/e2/03ff707d881d590c7adaed5e9d1979aed7e5e53fc1ed89035e5ed9f304af/opentelemetry_instrumentation_asgi-0.58b0.tar.gz", hash = "sha256:3ccc0c9c1c8c71e8d9da5945c6dcd9c0c8d147839f208536b7042c6dd98e65c9", size = 25116, upload-time = "2025-09-11T11:42:18.437Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/8c/71/a00884c6655387c70070138acbf79a6616ad5d4489680f40708d75b598a7/opentelemetry_instrumentation_asgi-0.58b0-py3-none-any.whl", hash = "sha256:508a6d79e333d648d2afee0e140b6e80eb5d443be183be58e81d9ff88373168a", size = 16798, upload-time = "2025-09-11T11:41:08.105Z" },
]
[[package]]
name = "opentelemetry-instrumentation-fastapi"
version = "0.58b0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "opentelemetry-instrumentation" },
{ name = "opentelemetry-instrumentation-asgi" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "opentelemetry-util-http" },
]
sdist = { url = "https://files.pythonhosted.org/packages/64/09/4f8fcab834af6b403e5e2d94bdfb2d0835ba8cd1049bcc156995f47b65fb/opentelemetry_instrumentation_fastapi-0.58b0.tar.gz", hash = "sha256:03da470d694116a0a40f4e76319e42f3ff9efc49abf804b2acc2c07f96661497", size = 24598, upload-time = "2025-09-11T11:42:35.325Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/45/fb/82de06eba54e5cb979274f073065ebc374794853502d342b5155073d1194/opentelemetry_instrumentation_fastapi-0.58b0-py3-none-any.whl", hash = "sha256:d89bfec69c9ffc5d9f3fe58655d6660a66b2bca863b9132712c06edcde68b6fa", size = 13460, upload-time = "2025-09-11T11:41:28.507Z" },
]
[[package]]
name = "opentelemetry-proto"
version = "1.37.0"
@@ -1234,15 +1192,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" },
]
[[package]]
name = "opentelemetry-util-http"
version = "0.58b0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c6/5f/02f31530faf50ef8a41ab34901c05cbbf8e9d76963ba2fb852b0b4065f4e/opentelemetry_util_http-0.58b0.tar.gz", hash = "sha256:de0154896c3472c6599311c83e0ecee856c4da1b17808d39fdc5cce5312e4d89", size = 9411, upload-time = "2025-09-11T11:43:05.602Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a5/a3/0a1430c42c6d34d8372a16c104e7408028f0c30270d8f3eb6cccf2e82934/opentelemetry_util_http-0.58b0-py3-none-any.whl", hash = "sha256:6c6b86762ed43025fbd593dc5f700ba0aa3e09711aedc36fd48a13b23d8cb1e7", size = 7652, upload-time = "2025-09-11T11:42:09.682Z" },
]
[[package]]
name = "packaging"
version = "25.0"
@@ -1360,6 +1309,47 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/07/d1/0a28c21707807c6aacd5dc9c3704b2aa1effbf37adebd8caeaf68b17a636/protobuf-6.33.0-py3-none-any.whl", hash = "sha256:25c9e1963c6734448ea2d308cfa610e692b801304ba0908d7bfa564ac5132995", size = 170477, upload-time = "2025-10-15T20:39:51.311Z" },
]
[[package]]
name = "psycopg2-binary"
version = "2.9.11"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c", size = 379620, upload-time = "2025-10-10T11:14:48.041Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d8/91/f870a02f51be4a65987b45a7de4c2e1897dd0d01051e2b559a38fa634e3e/psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4", size = 3756603, upload-time = "2025-10-10T11:11:52.213Z" },
{ url = "https://files.pythonhosted.org/packages/27/fa/cae40e06849b6c9a95eb5c04d419942f00d9eaac8d81626107461e268821/psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc", size = 3864509, upload-time = "2025-10-10T11:11:56.452Z" },
{ url = "https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a", size = 4411159, upload-time = "2025-10-10T11:12:00.49Z" },
{ url = "https://files.pythonhosted.org/packages/6f/a0/567f7ea38b6e1c62aafd58375665a547c00c608a471620c0edc364733e13/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e", size = 4468234, upload-time = "2025-10-10T11:12:04.892Z" },
{ url = "https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db", size = 4166236, upload-time = "2025-10-10T11:12:11.674Z" },
{ url = "https://files.pythonhosted.org/packages/3c/94/c1777c355bc560992af848d98216148be5f1be001af06e06fc49cbded578/psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757", size = 3983083, upload-time = "2025-10-30T02:55:15.73Z" },
{ url = "https://files.pythonhosted.org/packages/bd/42/c9a21edf0e3daa7825ed04a4a8588686c6c14904344344a039556d78aa58/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3", size = 3652281, upload-time = "2025-10-10T11:12:17.713Z" },
{ url = "https://files.pythonhosted.org/packages/12/22/dedfbcfa97917982301496b6b5e5e6c5531d1f35dd2b488b08d1ebc52482/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a", size = 3298010, upload-time = "2025-10-10T11:12:22.671Z" },
{ url = "https://files.pythonhosted.org/packages/66/ea/d3390e6696276078bd01b2ece417deac954dfdd552d2edc3d03204416c0c/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34", size = 3044641, upload-time = "2025-10-30T02:55:19.929Z" },
{ url = "https://files.pythonhosted.org/packages/12/9a/0402ded6cbd321da0c0ba7d34dc12b29b14f5764c2fc10750daa38e825fc/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d", size = 3347940, upload-time = "2025-10-10T11:12:26.529Z" },
{ url = "https://files.pythonhosted.org/packages/b1/d2/99b55e85832ccde77b211738ff3925a5d73ad183c0b37bcbbe5a8ff04978/psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d", size = 2714147, upload-time = "2025-10-10T11:12:29.535Z" },
{ url = "https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c", size = 3756572, upload-time = "2025-10-10T11:12:32.873Z" },
{ url = "https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee", size = 3864529, upload-time = "2025-10-10T11:12:36.791Z" },
{ url = "https://files.pythonhosted.org/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0", size = 4411242, upload-time = "2025-10-10T11:12:42.388Z" },
{ url = "https://files.pythonhosted.org/packages/10/04/6ca7477e6160ae258dc96f67c371157776564679aefd247b66f4661501a2/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766", size = 4468258, upload-time = "2025-10-10T11:12:48.654Z" },
{ url = "https://files.pythonhosted.org/packages/3c/7e/6a1a38f86412df101435809f225d57c1a021307dd0689f7a5e7fe83588b1/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3", size = 4166295, upload-time = "2025-10-10T11:12:52.525Z" },
{ url = "https://files.pythonhosted.org/packages/f2/7d/c07374c501b45f3579a9eb761cbf2604ddef3d96ad48679112c2c5aa9c25/psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f", size = 3983133, upload-time = "2025-10-30T02:55:24.329Z" },
{ url = "https://files.pythonhosted.org/packages/82/56/993b7104cb8345ad7d4516538ccf8f0d0ac640b1ebd8c754a7b024e76878/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4", size = 3652383, upload-time = "2025-10-10T11:12:56.387Z" },
{ url = "https://files.pythonhosted.org/packages/2d/ac/eaeb6029362fd8d454a27374d84c6866c82c33bfc24587b4face5a8e43ef/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c", size = 3298168, upload-time = "2025-10-10T11:13:00.403Z" },
{ url = "https://files.pythonhosted.org/packages/2b/39/50c3facc66bded9ada5cbc0de867499a703dc6bca6be03070b4e3b65da6c/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60", size = 3044712, upload-time = "2025-10-30T02:55:27.975Z" },
{ url = "https://files.pythonhosted.org/packages/9c/8e/b7de019a1f562f72ada81081a12823d3c1590bedc48d7d2559410a2763fe/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1", size = 3347549, upload-time = "2025-10-10T11:13:03.971Z" },
{ url = "https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa", size = 2714215, upload-time = "2025-10-10T11:13:07.14Z" },
{ url = "https://files.pythonhosted.org/packages/64/12/93ef0098590cf51d9732b4f139533732565704f45bdc1ffa741b7c95fb54/psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1", size = 3756567, upload-time = "2025-10-10T11:13:11.885Z" },
{ url = "https://files.pythonhosted.org/packages/7c/a9/9d55c614a891288f15ca4b5209b09f0f01e3124056924e17b81b9fa054cc/psycopg2_binary-2.9.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0deeb03da539fa3577fcb0b3f2554a97f7e5477c246098dbb18091a4a01c16f", size = 3864755, upload-time = "2025-10-10T11:13:17.727Z" },
{ url = "https://files.pythonhosted.org/packages/13/1e/98874ce72fd29cbde93209977b196a2edae03f8490d1bd8158e7f1daf3a0/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5", size = 4411646, upload-time = "2025-10-10T11:13:24.432Z" },
{ url = "https://files.pythonhosted.org/packages/5a/bd/a335ce6645334fb8d758cc358810defca14a1d19ffbc8a10bd38a2328565/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8", size = 4468701, upload-time = "2025-10-10T11:13:29.266Z" },
{ url = "https://files.pythonhosted.org/packages/44/d6/c8b4f53f34e295e45709b7568bf9b9407a612ea30387d35eb9fa84f269b4/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c", size = 4166293, upload-time = "2025-10-10T11:13:33.336Z" },
{ url = "https://files.pythonhosted.org/packages/4b/e0/f8cc36eadd1b716ab36bb290618a3292e009867e5c97ce4aba908cb99644/psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f", size = 3983184, upload-time = "2025-10-30T02:55:32.483Z" },
{ url = "https://files.pythonhosted.org/packages/53/3e/2a8fe18a4e61cfb3417da67b6318e12691772c0696d79434184a511906dc/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747", size = 3652650, upload-time = "2025-10-10T11:13:38.181Z" },
{ url = "https://files.pythonhosted.org/packages/76/36/03801461b31b29fe58d228c24388f999fe814dfc302856e0d17f97d7c54d/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f", size = 3298663, upload-time = "2025-10-10T11:13:44.878Z" },
{ url = "https://files.pythonhosted.org/packages/97/77/21b0ea2e1a73aa5fa9222b2a6b8ba325c43c3a8d54272839c991f2345656/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b", size = 3044737, upload-time = "2025-10-30T02:55:35.69Z" },
{ url = "https://files.pythonhosted.org/packages/67/69/f36abe5f118c1dca6d3726ceae164b9356985805480731ac6712a63f24f0/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d", size = 3347643, upload-time = "2025-10-10T11:13:53.499Z" },
{ url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" },
]
[[package]]
name = "pybars3"
version = "0.9.7"