diff --git a/pyproject.toml b/pyproject.toml index 90198238..8c73be1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "aiosqlite>=0.20.0", "greenlet>=3.1.1", "pydantic[email,timezone]>=2.10.3", + "icecream>=2.1.3", ] [project.optional-dependencies] diff --git a/src/basic_memory/fileio.py b/src/basic_memory/fileio.py new file mode 100644 index 00000000..e33c771d --- /dev/null +++ b/src/basic_memory/fileio.py @@ -0,0 +1,120 @@ +""" +File I/O operations for basic-memory. +Handles reading and writing entities and observations to the filesystem. +""" +from pathlib import Path + +from basic_memory.schemas import Entity, Observation + + +class FileOperationError(Exception): + """Raised when file operations fail""" + pass + + +class EntityNotFoundError(Exception): + """Raised when an entity cannot be found""" + pass + + +async def write_entity_file(entities_path: Path, entity: Entity) -> bool: + """ + Write entity to filesystem in markdown format. + + Args: + entities_path: Path to entities directory + entity: Entity to write + + Returns: + True if successful + + Raises: + FileOperationError: If file operations fail + """ + entity_path = entities_path / f"{entity.id}.md" + + # Handle directory creation separately + try: + entity_path.parent.mkdir(parents=True, exist_ok=True) + except Exception as e: + raise FileOperationError(f"Failed to create entity directory: {str(e)}") from e + + # Format entity data as markdown + content = [ + f"# {entity.name}\n", + f"type: {entity.entity_type}\n", + "\n", # Observations section + "## Observations\n", + ] + + # Add observations + for obs in entity.observations: + content.append(f"- {obs.content}\n") + + # Handle atomic write operation + temp_path = entity_path.with_suffix('.tmp') + try: + temp_path.write_text("".join(content)) + except Exception as e: + raise FileOperationError(f"Failed to write temporary entity file: {str(e)}") from e + + try: + temp_path.rename(entity_path) + except Exception as e: + raise FileOperationError(f"Failed to finalize entity file: {str(e)}") from e + + return True + + +async def read_entity_file(entities_path: Path, entity_id: str) -> Entity: + """ + Read entity data from filesystem. + + Args: + entities_path: Path to entities directory + entity_id: ID of entity to read + + Returns: + Entity object + + Raises: + EntityNotFoundError: If entity file doesn't exist + FileOperationError: If file operations fail + """ + entity_path = entities_path / f"{entity_id}.md" + if not entity_path.exists(): + raise EntityNotFoundError(f"Entity file not found: {entity_id}") + + try: + content = entity_path.read_text().split("\n") + except Exception as e: + raise FileOperationError(f"Failed to read entity file: {str(e)}") from e + + # Parse markdown content + # First line should be "# Name" + name = content[0].lstrip("# ").strip() + + # Parse metadata (type) + entity_type = "" + observations = [] + + # Parse content sections + in_observations = False + for line in content[1:]: # Skip the title line + line = line.strip() + if not line: + continue + + if line.startswith("type: "): + entity_type = line.replace("type: ", "").strip() + elif line == "## Observations": + in_observations = True + elif in_observations and line.startswith("- "): + observations.append(Observation(content=line[2:])) + + return Entity( + id=entity_id, + name=name, + entity_type=entity_type, + observations=observations + ) diff --git a/src/basic_memory/models.py b/src/basic_memory/models.py index 53ad0d8c..fc52fa17 100644 --- a/src/basic_memory/models.py +++ b/src/basic_memory/models.py @@ -1,6 +1,8 @@ +import typing from datetime import datetime, UTC from typing import List, Optional from sqlalchemy import String, DateTime, ForeignKey, Text, TypeDecorator +from sqlalchemy.orm.exc import DetachedInstanceError from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase from sqlalchemy.ext.asyncio import AsyncAttrs @@ -81,6 +83,9 @@ class Entity(Base): cascade="all, delete-orphan" ) + def __repr__(self) -> str: + return f"Entity(id='{self.id}', name='{self.name}', type='{self.entity_type}')" + class Observation(Base): """ @@ -111,6 +116,10 @@ class Observation(Base): back_populates="observations" ) + def __repr__(self) -> str: + content = self.content[:50] + "..." if len(self.content) > 50 else self.content + return f"Observation(id='{self.id}', entity='{self.entity_id}', content='{content}')" + class Relation(Base): """ @@ -147,4 +156,7 @@ class Relation(Base): "Entity", foreign_keys=[to_id], back_populates="incoming_relations" - ) \ No newline at end of file + ) + + def __repr__(self) -> str: + return f"Relation(from='{self.from_id}', type='{self.relation_type}', to='{self.to_id}')" \ No newline at end of file diff --git a/src/basic_memory/services.py b/src/basic_memory/services.py index db858233..ba24386b 100644 --- a/src/basic_memory/services.py +++ b/src/basic_memory/services.py @@ -1,10 +1,17 @@ from datetime import datetime, UTC from pathlib import Path from typing import Optional +from uuid import uuid4 +from sqlalchemy import and_, select, delete from basic_memory.models import Entity as DbEntity # Rename to avoid confusion -from basic_memory.repository import EntityRepository +from basic_memory.models import Observation as DbObservation +from basic_memory.repository import EntityRepository, ObservationRepository from basic_memory.schemas import Entity, Observation +from basic_memory.fileio import ( + read_entity_file, write_entity_file, + FileOperationError, EntityNotFoundError +) class ServiceError(Exception): @@ -12,21 +19,11 @@ class ServiceError(Exception): pass -class FileOperationError(ServiceError): - """Raised when file operations fail""" - pass - - class DatabaseSyncError(ServiceError): """Raised when database sync fails""" pass -class EntityNotFoundError(ServiceError): - """Raised when an entity cannot be found""" - pass - - class EntityService: """ Service for managing entities in the filesystem and database. @@ -38,82 +35,6 @@ class EntityService: self.entity_repo = entity_repo self.entities_path = project_path / "entities" - async def _write_entity_file(self, entity: Entity) -> bool: - """Write entity to filesystem in markdown format.""" - entity_path = self.entities_path / f"{entity.id}.md" - - # Handle directory creation separately - try: - entity_path.parent.mkdir(parents=True, exist_ok=True) - except Exception as e: - raise FileOperationError(f"Failed to create entity directory: {str(e)}") from e - - # Format entity data as markdown - content = [ - f"# {entity.name}\n", - f"type: {entity.entity_type}\n", - "\n", # Observations section - "## Observations\n", - ] - - # Add observations - for obs in entity.observations: - content.append(f"- {obs.content}\n") - - # Handle atomic write operation - temp_path = entity_path.with_suffix('.tmp') - try: - temp_path.write_text("".join(content)) - except Exception as e: - raise FileOperationError(f"Failed to write temporary entity file: {str(e)}") from e - - try: - temp_path.rename(entity_path) - except Exception as e: - raise FileOperationError(f"Failed to finalize entity file: {str(e)}") from e - - return True - - async def _read_entity_file(self, entity_id: str) -> Entity: - """Read entity data from filesystem.""" - entity_path = self.entities_path / f"{entity_id}.md" - if not entity_path.exists(): - raise EntityNotFoundError(f"Entity file not found: {entity_id}") - - try: - content = entity_path.read_text().split("\n") - except Exception as e: - raise FileOperationError(f"Failed to read entity file: {str(e)}") from e - - # Parse markdown content - # First line should be "# Name" - name = content[0].lstrip("# ").strip() - - # Parse metadata (type) - entity_type = "" - observations = [] - - # Parse content sections - in_observations = False - for line in content[1:]: # Skip the title line - line = line.strip() - if not line: - continue - - if line.startswith("type: "): - entity_type = line.replace("type: ", "").strip() - elif line == "## Observations": - in_observations = True - elif in_observations and line.startswith("- "): - observations.append(Observation(content=line[2:])) - - return Entity( - id=entity_id, - name=name, - entity_type=entity_type, - observations=observations - ) - async def _update_db_index(self, entity: Entity) -> DbEntity: """Update database index with entity data.""" entity_data = { @@ -145,7 +66,7 @@ class EntityService: ) # Step 1: Write to filesystem (source of truth) - await self._write_entity_file(entity) + await write_entity_file(self.entities_path, entity) # Step 2: Update database index await self._update_db_index(entity) @@ -155,7 +76,7 @@ class EntityService: async def get_entity(self, entity_id: str) -> Entity: """Get entity by ID, reading from filesystem first.""" # Read from filesystem (source of truth) - entity = await self._read_entity_file(entity_id) + entity = await read_entity_file(self.entities_path, entity_id) # Update database index await self._update_db_index(entity) @@ -187,7 +108,118 @@ class EntityService: for entity_file in entity_files: try: - entity = await self._read_entity_file(entity_file.stem) + entity = await read_entity_file(self.entities_path, entity_file.stem) await self._update_db_index(entity) except Exception as e: print(f"Warning: Failed to reindex {entity_file}: {str(e)}") + + +class ObservationService: + """ + Service for managing observations in the filesystem and database. + Follows the "filesystem is source of truth" principle. + + Observations are stored in entity markdown files and indexed in the database + for efficient querying. + """ + + def __init__(self, project_path: Path, observation_repo: ObservationRepository): + self.project_path = project_path + self.entities_path = project_path / "entities" + self.observation_repo = observation_repo + + async def add_observation(self, entity: Entity, content: str, + context: Optional[str] = None) -> Observation: + """ + Add a new observation to an entity. + + Args: + entity: Entity to add observation to + content: Content of the observation + context: Optional context for the observation + + Returns: + The created Observation + + Raises: + FileOperationError: If file operations fail + DatabaseSyncError: If database sync fails + """ + # Create new observation + observation = Observation(content=content) + entity.observations.append(observation) + + # Update filesystem first (source of truth) + await write_entity_file(self.entities_path, entity) + + # Update database index + try: + db_observation = await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': content, + 'context': context, + 'created_at': datetime.now(UTC) + }) + return observation + except Exception as e: + raise DatabaseSyncError(f"Failed to sync observation to database: {str(e)}") from e + + async def search_observations(self, query: str) -> list[Observation]: + """ + Search for observations across all entities. + + Args: + query: Text to search for in observation content + + Returns: + List of matching observations with their entity contexts + """ + result = await self.observation_repo.execute_query( + select(DbObservation).filter( + DbObservation.content.contains(query) + ) + ) + return [ + Observation(content=obs.content) + for obs in result.scalars().all() + ] + + async def get_observations_by_context(self, context: str) -> list[Observation]: + """Get all observations with a specific context.""" + db_observations = await self.observation_repo.find_by_context(context) + return [ + Observation(content=obs.content) + for obs in db_observations + ] + + async def rebuild_observation_index(self) -> None: + """ + Rebuild the observation database index from filesystem contents. + Used for recovery or ensuring sync. + """ + # List all entity files + if not self.entities_path.exists(): + return + + try: + entity_files = list(self.entities_path.glob("*.md")) + except Exception as e: + raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e + + # Clear existing observation index + await self.observation_repo.execute_query(delete(DbObservation)) + + # Rebuild from each entity file + for entity_file in entity_files: + try: + entity = await read_entity_file(self.entities_path, entity_file.stem) + for obs in entity.observations: + await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': obs.content, + 'created_at': datetime.now(UTC) + }) + except Exception as e: + print(f"Warning: Failed to reindex observations for {entity_file}: {str(e)}") diff --git a/tests/test_observation_service.py b/tests/test_observation_service.py new file mode 100644 index 00000000..aba64a5b --- /dev/null +++ b/tests/test_observation_service.py @@ -0,0 +1,240 @@ +import pytest +import pytest_asyncio +from datetime import datetime +from pathlib import Path +import tempfile +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from sqlalchemy.pool import StaticPool +from sqlalchemy import delete + +from basic_memory.models import Base, Entity as DbEntity, Observation as DbObservation +from basic_memory.repository import EntityRepository, ObservationRepository +from basic_memory.services import ( + EntityService, ObservationService, + FileOperationError, DatabaseSyncError, ServiceError +) +from basic_memory.schemas import Entity, Observation +from basic_memory.fileio import read_entity_file + +pytestmark = pytest.mark.asyncio + +@pytest_asyncio.fixture(scope="function") +async def engine(): + """Create an async engine using in-memory SQLite database""" + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + echo=False, + poolclass=StaticPool, + connect_args={"check_same_thread": False} + ) + + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + try: + yield engine + finally: + await engine.dispose() + +@pytest_asyncio.fixture(scope="function") +async def session(engine): + """Create an async session factory and yield a session""" + async_session = async_sessionmaker(engine, expire_on_commit=False) + async with async_session() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + +@pytest_asyncio.fixture +async def entity_repo(session): + """Create an EntityRepository instance.""" + return EntityRepository(session, DbEntity) + +@pytest_asyncio.fixture +async def observation_repo(session): + """Create an ObservationRepository instance.""" + return ObservationRepository(session, DbObservation) + +@pytest_asyncio.fixture +async def entity_service(session, entity_repo): + """Fixture providing initialized EntityService with temp directories.""" + with tempfile.TemporaryDirectory() as temp_dir: + project_path = Path(temp_dir) / "test-project" + entities_path = project_path / "entities" + entities_path.mkdir(parents=True) + + service = EntityService(project_path, entity_repo) + yield service + +@pytest_asyncio.fixture +async def observation_service(session, observation_repo): + """Fixture providing initialized ObservationService.""" + with tempfile.TemporaryDirectory() as temp_dir: + project_path = Path(temp_dir) / "test-project" + entities_path = project_path / "entities" + entities_path.mkdir(parents=True) + + service = ObservationService(project_path, observation_repo) + yield service + +@pytest_asyncio.fixture +async def test_entity(entity_service): + """Create a test entity for observation operations.""" + return await entity_service.create_entity( + name="Test Entity", + entity_type="test", + ) + +# Happy Path Tests + +async def test_add_observation_success(observation_service, test_entity): + """Test successful observation addition.""" + # Act + observation = await observation_service.add_observation( + entity=test_entity, + content="New observation", + context="test-context" + ) + + # Assert + assert isinstance(observation, Observation) + assert observation.content == "New observation" + + # Verify file update + entity = await read_entity_file(observation_service.entities_path, test_entity.id) + assert len(entity.observations) == 1 + assert any(obs.content == "New observation" for obs in entity.observations) + + # Verify database index + db_observations = await observation_service.observation_repo.find_by_entity(test_entity.id) + assert len(db_observations) == 1 + assert any(obs.content == "New observation" and obs.context == "test-context" + for obs in db_observations) + +async def test_search_observations(observation_service, test_entity): + """Test searching observations across entities.""" + # Arrange + await observation_service.add_observation(test_entity, "Unique test content") + await observation_service.add_observation(test_entity, "Other content") + + # Act + results = await observation_service.search_observations("unique") + + # Assert + assert len(results) == 1 + assert results[0].content == "Unique test content" + +async def test_get_observations_by_context(observation_service, test_entity): + """Test retrieving observations by context.""" + # Arrange + await observation_service.add_observation( + test_entity, + "Context observation", + context="test-context" + ) + await observation_service.add_observation( + test_entity, + "Other observation", + context="other-context" + ) + + # Act + results = await observation_service.get_observations_by_context("test-context") + + # Assert + assert len(results) == 1 + assert results[0].content == "Context observation" + +# Error Path Tests + +async def test_file_operation_error(observation_service, test_entity, monkeypatch): + """Test handling of file operation errors.""" + async def mock_write(*args, **kwargs): + raise FileOperationError("Mock file error") + monkeypatch.setattr('basic_memory.services.write_entity_file', mock_write) + + with pytest.raises(FileOperationError): + await observation_service.add_observation( + test_entity, + "Test observation" + ) + +async def test_database_sync_error(observation_service, test_entity, monkeypatch): + """Test handling of database sync errors.""" + async def mock_create(*args, **kwargs): + raise Exception("Mock DB error") + monkeypatch.setattr(observation_service.observation_repo, "create", mock_create) + + with pytest.raises(DatabaseSyncError): + await observation_service.add_observation( + test_entity, + "Test observation" + ) + +# Recovery Tests + +async def test_rebuild_observation_index(observation_service, test_entity): + """Test rebuilding observation index from filesystem.""" + # Arrange - Add observations and clear database + await observation_service.add_observation(test_entity, "Test observation 1") + await observation_service.add_observation(test_entity, "Test observation 2") + + # Clear database but keep files + await observation_service.observation_repo.execute_query(delete(DbObservation)) + + # Act + await observation_service.rebuild_observation_index() + + # Assert + db_observations = await observation_service.observation_repo.find_by_entity(test_entity.id) + assert len(db_observations) == 2 + observation_contents = {obs.content for obs in db_observations} + assert observation_contents == { + "Test observation 1", + "Test observation 2" + } + +# Edge Cases + +async def test_observation_with_special_characters(observation_service, test_entity): + """Test handling observations with special characters.""" + content = "Test & observation with @#$% special chars!" + observation = await observation_service.add_observation( + test_entity, + content + ) + assert observation.content == content + + # Verify file content + entity = await read_entity_file(observation_service.entities_path, test_entity.id) + assert any(obs.content == content for obs in entity.observations) + + +async def test_very_long_observation(observation_service, test_entity): + """Test handling very long observation content.""" + long_content = "Very long observation " * 100 # ~1800 characters + observation = await observation_service.add_observation( + test_entity, + long_content + ) + assert observation.content == long_content + + # Debug print actual file content + entity_path = observation_service.entities_path / f"{test_entity.id}.md" + print("File content:", entity_path.read_text()) + + # Verify file content + entity = await read_entity_file(observation_service.entities_path, test_entity.id) + print("Loaded observations:", [obs.content for obs in entity.observations]) + assert observation.content.rstrip() == entity.observations[0].content.rstrip() + + +# TODO: Add concurrent operation tests once we have proper session management +# Currently SQLAlchemy sessions are not safe for concurrent use. +# We'll need either: +# 1. Session per operation pattern +# 2. Higher level concurrency handling (e.g., API layer) +# See error: IllegalStateChangeError with concurrent session usage \ No newline at end of file diff --git a/uv.lock b/uv.lock index 26a9786a..0688db93 100644 --- a/uv.lock +++ b/uv.lock @@ -22,6 +22,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, ] +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918 }, +] + [[package]] name = "basic-memory" version = "0.1.0" @@ -29,6 +38,7 @@ source = { editable = "." } dependencies = [ { name = "aiosqlite" }, { name = "greenlet" }, + { name = "icecream" }, { name = "pydantic", extra = ["email", "timezone"] }, { name = "pyyaml" }, { name = "rich" }, @@ -49,6 +59,7 @@ requires-dist = [ { name = "aiosqlite", specifier = ">=0.20.0" }, { name = "black", marker = "extra == 'dev'", specifier = ">=23.11.0" }, { name = "greenlet", specifier = ">=3.1.1" }, + { name = "icecream", specifier = ">=2.1.3" }, { name = "pydantic", extras = ["email", "timezone"], specifier = ">=2.10.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4.3" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" }, @@ -164,6 +175,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521 }, ] +[[package]] +name = "executing" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e3/7d45f492c2c4a0e8e0fad57d081a7c8a0286cdd86372b070cca1ec0caa1e/executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab", size = 977485 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/fd/afcd0496feca3276f509df3dbd5dae726fcc756f1a08d9e25abe1733f962/executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf", size = 25805 }, +] + [[package]] name = "greenlet" version = "3.1.1" @@ -197,6 +217,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/38/08cc303ddddc4b3d7c628c3039a61a3aae36c241ed01393d00c2fd663473/greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6", size = 1142112 }, ] +[[package]] +name = "icecream" +version = "2.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "colorama" }, + { name = "executing" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/8b/ae6ebc9fc423f9397a0982990c86f1fe94077df729ef452c9a847fb16ae6/icecream-2.1.3.tar.gz", hash = "sha256:0aa4a7c3374ec36153a1d08f81e3080e83d8ac1eefd97d2f4fe9544e8f9b49de", size = 14722 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/4e/21e309c7087695cf500a1827597f8510641f2c9a50ed7741bf7fc38736ff/icecream-2.1.3-py2.py3-none-any.whl", hash = "sha256:757aec31ad4488b949bc4f499d18e6e5973c40cc4d4fc607229e78cfaec94c34", size = 8425 }, +] + [[package]] name = "idna" version = "3.10"