diff --git a/docker-compose-postgres.yml b/docker-compose-postgres.yml new file mode 100644 index 00000000..515e650b --- /dev/null +++ b/docker-compose-postgres.yml @@ -0,0 +1,42 @@ +# Docker Compose configuration for Basic Memory with PostgreSQL +# Use this for local development and testing with Postgres backend +# +# Usage: +# docker-compose -f docker-compose-postgres.yml up -d +# docker-compose -f docker-compose-postgres.yml down + +services: + postgres: + image: postgres:17 + container_name: basic-memory-postgres + environment: + # Local development/test credentials - NOT for production + # These values are referenced by tests and justfile commands + POSTGRES_DB: basic_memory + POSTGRES_USER: basic_memory_user + POSTGRES_PASSWORD: dev_password # Simple password for local testing only + ports: + - "5433:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U basic_memory_user -d basic_memory"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + +volumes: + # Named volume for Postgres data + postgres_data: + driver: local + + # Named volume for persistent configuration + # Database will be stored in Postgres, not in this volume + basic-memory-config: + driver: local + +# Network configuration (optional) +# networks: +# basic-memory-net: +# driver: bridge diff --git a/justfile b/justfile index 55ab1761..44241869 100644 --- a/justfile +++ b/justfile @@ -51,7 +51,7 @@ test-int-sqlite: # Note: Uses timeout due to FastMCP Client + asyncpg cleanup hang (tests pass, process hangs on exit) # See: https://github.com/jlowin/fastmcp/issues/1311 test-int-postgres: - timeout --signal=KILL 300 bash -c 'BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov test-int' || test $? -eq 137 + timeout --signal=KILL 600 bash -c 'BASIC_MEMORY_TEST_POSTGRES=1 uv run pytest -p pytest_mock -v --no-cov test-int' || test $? -eq 137 # Reset Postgres test database (drops and recreates schema) # Useful when Alembic migration state gets out of sync during development diff --git a/src/basic_memory/api/routers/resource_router.py b/src/basic_memory/api/routers/resource_router.py index 1a658b28..2a096555 100644 --- a/src/basic_memory/api/routers/resource_router.py +++ b/src/basic_memory/api/routers/resource_router.py @@ -185,21 +185,17 @@ async def write_resource( else: content_str = str(content) - # Get full file path - full_path = Path(f"{config.home}/{file_path}") - - # Ensure parent directory exists - full_path.parent.mkdir(parents=True, exist_ok=True) - - # Write content to file - checksum = await file_service.write_file(full_path, content_str) + # Cloud compatibility: do not assume a local filesystem path structure. + # Delegate directory creation + writes to the configured FileService (local or S3). + await file_service.ensure_directory(Path(file_path).parent) + checksum = await file_service.write_file(file_path, content_str) # Get file info - file_metadata = await file_service.get_file_metadata(full_path) + file_metadata = await file_service.get_file_metadata(file_path) # Determine file details file_name = Path(file_path).name - content_type = file_service.content_type(full_path) + content_type = file_service.content_type(file_path) entity_type = "canvas" if file_path.endswith(".canvas") else "file" diff --git a/src/basic_memory/api/routers/utils.py b/src/basic_memory/api/routers/utils.py index 0e7509c4..b4211a67 100644 --- a/src/basic_memory/api/routers/utils.py +++ b/src/basic_memory/api/routers/utils.py @@ -27,7 +27,9 @@ async def to_graph_context( # First pass: collect all entity IDs needed for relations entity_ids_needed: set[int] = set() for context_item in context_result.results: - for item in [context_item.primary_result] + context_item.observations + context_item.related_results: + for item in ( + [context_item.primary_result] + context_item.observations + context_item.related_results + ): if item.type == SearchItemType.RELATION: if item.from_id: # pyright: ignore entity_ids_needed.add(item.from_id) # pyright: ignore diff --git a/src/basic_memory/api/v2/routers/resource_router.py b/src/basic_memory/api/v2/routers/resource_router.py index 045f5238..f92c1794 100644 --- a/src/basic_memory/api/v2/routers/resource_router.py +++ b/src/basic_memory/api/v2/routers/resource_router.py @@ -135,21 +135,17 @@ async def create_resource( f"Use PUT /resource/{existing_entity.id} to update it.", ) - # Get full file path - full_path = Path(f"{config.home}/{data.file_path}") - - # Ensure parent directory exists - full_path.parent.mkdir(parents=True, exist_ok=True) - - # Write content to file - checksum = await file_service.write_file(full_path, data.content) + # Cloud compatibility: avoid assuming a local filesystem path. + # Delegate directory creation + writes to FileService (local or S3). + await file_service.ensure_directory(Path(data.file_path).parent) + checksum = await file_service.write_file(data.file_path, data.content) # Get file info - file_metadata = await file_service.get_file_metadata(full_path) + file_metadata = await file_service.get_file_metadata(data.file_path) # Determine file details file_name = Path(data.file_path).name - content_type = file_service.content_type(full_path) + content_type = file_service.content_type(data.file_path) entity_type = "canvas" if data.file_path.endswith(".canvas") else "file" # Create a new entity model @@ -234,30 +230,27 @@ async def update_resource( "Path must be relative and stay within project boundaries.", ) - # Get full paths - new_full_path = Path(f"{config.home}/{target_file_path}") - # If moving file, handle the move if data.file_path and data.file_path != entity.file_path: - # Ensure new parent directory exists - new_full_path.parent.mkdir(parents=True, exist_ok=True) + # Ensure new parent directory exists (no-op for S3) + await file_service.ensure_directory(Path(target_file_path).parent) # If old file exists, remove it via file_service (for cloud compatibility) if await file_service.exists(entity.file_path): await file_service.delete_file(entity.file_path) else: # Ensure directory exists for in-place update - new_full_path.parent.mkdir(parents=True, exist_ok=True) + await file_service.ensure_directory(Path(target_file_path).parent) # Write content to target file - checksum = await file_service.write_file(new_full_path, data.content) + checksum = await file_service.write_file(target_file_path, data.content) # Get file info - file_metadata = await file_service.get_file_metadata(new_full_path) + file_metadata = await file_service.get_file_metadata(target_file_path) # Determine file details file_name = Path(target_file_path).name - content_type = file_service.content_type(new_full_path) + content_type = file_service.content_type(target_file_path) entity_type = "canvas" if target_file_path.endswith(".canvas") else "file" # Update entity diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 9c9635cb..e0a06c50 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -792,23 +792,20 @@ class EntityService(BaseService[EntityModel]): raise ValueError(f"Invalid destination path: {destination_path}") # 3. Validate paths - source_file = project_config.home / current_path - destination_file = project_config.home / destination_path - - # Validate source exists - if not source_file.exists(): + # NOTE: In tenantless/cloud mode, we cannot rely on local filesystem paths. + # Use FileService for existence checks and moving. + if not await self.file_service.exists(current_path): raise ValueError(f"Source file not found: {current_path}") - # Check if destination already exists - if destination_file.exists(): + if await self.file_service.exists(destination_path): raise ValueError(f"Destination already exists: {destination_path}") try: - # 4. Create destination directory if needed - destination_file.parent.mkdir(parents=True, exist_ok=True) + # 4. Ensure destination directory if needed (no-op for S3) + await self.file_service.ensure_directory(Path(destination_path).parent) - # 5. Move physical file - source_file.rename(destination_file) + # 5. Move physical file via FileService (filesystem rename or cloud move) + await self.file_service.move_file(current_path, destination_path) logger.info(f"Moved file: {current_path} -> {destination_path}") # 6. Prepare database updates @@ -847,12 +844,14 @@ class EntityService(BaseService[EntityModel]): except Exception as e: # Rollback: try to restore original file location if move succeeded - if destination_file.exists() and not source_file.exists(): - try: - destination_file.rename(source_file) + try: + if await self.file_service.exists( + destination_path + ) and not await self.file_service.exists(current_path): + await self.file_service.move_file(destination_path, current_path) logger.info(f"Rolled back file move: {destination_path} -> {current_path}") - except Exception as rollback_error: # pragma: no cover - logger.error(f"Failed to rollback file move: {rollback_error}") + except Exception as rollback_error: # pragma: no cover + logger.error(f"Failed to rollback file move: {rollback_error}") # Re-raise the original error with context raise ValueError(f"Move failed: {str(e)}") from e diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index 55dacb71..4b738372 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -312,6 +312,43 @@ class FileService: full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj full_path.unlink(missing_ok=True) + async def move_file(self, source: FilePath, destination: FilePath) -> None: + """Move/rename a file from source to destination. + + This method abstracts the underlying storage (filesystem vs cloud). + Default implementation uses atomic filesystem rename, but cloud-backed + implementations (e.g., S3) can override to copy+delete. + + Args: + source: Source path (relative to base_path or absolute) + destination: Destination path (relative to base_path or absolute) + + Raises: + FileOperationError: If the move fails + """ + # Convert strings to Paths and resolve relative paths against base_path + src_obj = self.base_path / source if isinstance(source, str) else source + dst_obj = self.base_path / destination if isinstance(destination, str) else destination + src_full = src_obj if src_obj.is_absolute() else self.base_path / src_obj + dst_full = dst_obj if dst_obj.is_absolute() else self.base_path / dst_obj + + try: + # Ensure destination directory exists + await self.ensure_directory(dst_full.parent) + + # Use semaphore for concurrency control and run blocking rename in executor + async with self._file_semaphore: + loop = asyncio.get_event_loop() + await loop.run_in_executor(None, lambda: src_full.rename(dst_full)) + except Exception as e: + logger.exception( + "File move error", + source=str(src_full), + destination=str(dst_full), + error=str(e), + ) + raise FileOperationError(f"Failed to move file {source} -> {destination}: {e}") + async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str: """Update frontmatter fields in a file while preserving all content. diff --git a/tests/utils/test_timezone_utils.py b/tests/utils/test_timezone_utils.py index f61d40f8..5e359e66 100644 --- a/tests/utils/test_timezone_utils.py +++ b/tests/utils/test_timezone_utils.py @@ -2,7 +2,6 @@ from datetime import datetime, timezone -import pytest from basic_memory.utils import ensure_timezone_aware