diff --git a/docs/obsidian-test.md b/docs/obsidian-test.md deleted file mode 100644 index 8e878b39..00000000 --- a/docs/obsidian-test.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -id: 5 -created: '2024-12-24T02:30:29.343588+00:00' -modified: '2024-12-24T02:30:29.343588+00:00' -type: test -tags: -- obsidian -- markdown -- documentation -created_by: Claude -status: draft ---- - -# Obsidian Test Document - -This is a test of how documents appear in Obsidian's interface. - -## Links and Tags -We can use: -- Standard markdown links like [Basic Memory](basic-memory) -- Tags like #test #documentation -- Embeds like ![[basic-memory]] - -## Features to Test -### Knowledge Graph -This document should show up in the knowledge graph with connections to: -- [[Basic_Memory]] project -- [[Knowledge_Graph_Structure]] which implements it -- [[Development_Process]] that guides it - -### Backlinks -Any document that links to this one should appear in the backlinks panel. - -### YAML Frontmatter -Obsidian should display the frontmatter cleanly at the top of the document. - -### Code Blocks -```python -def test_function(): - """Code blocks should have syntax highlighting""" - print("Testing display") -``` - -### Callouts -> [!NOTE] -> Obsidian supports special callout blocks -> They help organize important information - -### Task Lists -- [x] Create test document -- [x] Add various markdown features -- [ ] View in Obsidian -- [ ] Check graph visualization \ No newline at end of file diff --git a/src/basic_memory/markdown/knowledge_writer.py b/src/basic_memory/markdown/knowledge_writer.py index 002efcc6..6fd8376a 100644 --- a/src/basic_memory/markdown/knowledge_writer.py +++ b/src/basic_memory/markdown/knowledge_writer.py @@ -1,6 +1,5 @@ """Writer for knowledge entity markdown files.""" -from datetime import datetime, UTC from typing import Optional, Dict, Any import yaml @@ -14,12 +13,11 @@ class KnowledgeWriter: async def format_frontmatter(self, entity: EntityModel) -> dict: """Generate frontmatter metadata for entity.""" - now = datetime.now(UTC).isoformat() return { "type": entity.entity_type, - "id": entity.id, - "created": now, - "modified": now + "id": entity.path_id, + "created": entity.created_at.isoformat(), + "modified": entity.updated_at.isoformat(), } async def format_metadata(self, metadata: Optional[Dict[str, Any]] = None) -> str: @@ -40,37 +38,50 @@ class KnowledgeWriter: logger.warning(f"Failed to format metadata YAML: {e}") return "" # Skip metadata on error - async def format_content(self, entity: EntityModel, metadata: Optional[Dict[str, Any]] = None) -> str: + async def format_content( + self, entity: EntityModel, metadata: Optional[Dict[str, Any]] = None + ) -> str: """Format entity content as markdown.""" sections = [ - f"# {entity.name}\n" + f"# {entity.name}\n", + "", # Empty line after name ] if entity.description: - sections.extend([ - entity.description, - "" - ]) - + sections.extend([entity.description, ""]) + if entity.observations: - sections.extend([ - "## Observations", - *[f"- {obs.content}" for obs in entity.observations], - "" - ]) + sections.extend( + [ + "## Observations", + "", + "", # Empty line after format comment + *[ + f"- [{obs.category}] {obs.content}" + + (f" ({obs.context})" if obs.context else "") + for obs in entity.observations + ], + "", + ] + ) # Format outgoing and incoming relations separately if entity.to_relations or entity.from_relations: - sections.append("## Relations") - + sections.extend( + [ + "## Relations", + "", # Empty line after format comment + ] + ) + # Outgoing relations for rel in entity.to_relations: sections.append(f"- [[{rel.from_entity.name}]] {rel.relation_type}") - + # Incoming relations for rel in entity.from_relations: sections.append(f"- [[{rel.to_entity.name}]] {rel.relation_type}") - + sections.append("") if metadata: diff --git a/src/basic_memory/models/__init__.py b/src/basic_memory/models/__init__.py index 0f31fa50..31cd4667 100644 --- a/src/basic_memory/models/__init__.py +++ b/src/basic_memory/models/__init__.py @@ -2,12 +2,13 @@ from basic_memory.models.base import Base from basic_memory.models.documents import Document -from basic_memory.models.knowledge import Entity, Observation, Relation +from basic_memory.models.knowledge import Entity, Observation, Relation, ObservationCategory __all__ = [ 'Base', 'Document', 'Entity', 'Observation', + 'ObservationCategory', 'Relation' ] \ No newline at end of file diff --git a/src/basic_memory/repository/observation_repository.py b/src/basic_memory/repository/observation_repository.py index bd502eee..b147d79b 100644 --- a/src/basic_memory/repository/observation_repository.py +++ b/src/basic_memory/repository/observation_repository.py @@ -26,3 +26,15 @@ class ObservationRepository(Repository[Observation]): query = select(Observation).filter(Observation.context == context) result = await self.execute_query(query) return result.scalars().all() + + async def find_by_category(self, category: str) -> Sequence[Observation]: + """Find observations with a specific context.""" + query = select(Observation).filter(Observation.category == category) + result = await self.execute_query(query) + return result.scalars().all() + + async def observation_categories(self) -> Sequence[str]: + """Return a list of all observation categories.""" + query = select(Observation.category).distinct() + result = await self.execute_query(query) + return result.scalars().all() diff --git a/src/basic_memory/schemas/request.py b/src/basic_memory/schemas/request.py index bc9ff869..3b9713f9 100644 --- a/src/basic_memory/schemas/request.py +++ b/src/basic_memory/schemas/request.py @@ -127,6 +127,7 @@ class SearchNodesRequest(BaseModel): - Partial word matches - Returns full entity objects with relations - Includes all matching entities + - If a category is specified, only entities with that category are returned Example Queries: - "memory" - Find entities related to memory systems @@ -143,6 +144,7 @@ class SearchNodesRequest(BaseModel): """ query: Annotated[str, MinLen(1), MaxLen(200)] + category: Optional[ObservationCategory] = None class OpenNodesRequest(BaseModel): diff --git a/src/basic_memory/services/knowledge/observations.py b/src/basic_memory/services/knowledge/observations.py index 50fb0e58..11335922 100644 --- a/src/basic_memory/services/knowledge/observations.py +++ b/src/basic_memory/services/knowledge/observations.py @@ -7,6 +7,7 @@ from loguru import logger from basic_memory.models import Entity as EntityModel from basic_memory.services.exceptions import EntityNotFoundError from basic_memory.services.observation_service import ObservationService +from basic_memory.schemas.request import ObservationCreate from .relations import RelationOperations @@ -18,9 +19,22 @@ class ObservationOperations(RelationOperations): self.observation_service = observation_service async def add_observations( - self, path_id: str, observations: List[str], context: str | None = None + self, + path_id: str, + observations: List[ObservationCreate], + context: str | None = None ) -> EntityModel: - """Add observations to entity and update its file.""" + """Add observations to entity and update its file. + + Observations are added with their categories and written to both + the database and markdown file. The file format is: + - [category] Content text #tag1 #tag2 (optional context) + + Args: + path_id: Entity path ID + observations: List of observations with categories + context: Optional shared context for all observations + """ logger.debug(f"Adding observations to entity {path_id}") try: @@ -33,25 +47,34 @@ class ObservationOperations(RelationOperations): await self.observation_service.add_observations(entity.id, observations, context) # Get updated entity - updated_entity = await self.entity_service.get_by_path_id(path_id) + entity = await self.entity_service.get_by_path_id(path_id) # Write updated file and checksum _, checksum = await self.write_entity_file(entity) await self.entity_service.update_entity(path_id, {"checksum": checksum}) - # query to fetch all relations + # Return final entity with all updates and relations return await self.entity_service.get_by_path_id(path_id) except Exception as e: logger.error(f"Failed to add observations: {e}") raise - async def delete_observations(self, path_id: str, observations: List[str]) -> EntityModel: - """Delete observations from entity and update its file.""" + async def delete_observations( + self, + path_id: str, + observations: List[str] + ) -> EntityModel: + """Delete observations from entity and update its file. + + Args: + path_id: Entity path ID + observations: List of observation contents to delete + """ logger.debug(f"Deleting observations from entity {path_id}") try: - # Get updated entity + # Get entity entity = await self.entity_service.get_by_path_id(path_id) if not entity: raise EntityNotFoundError(f"Entity not found: {path_id}") @@ -63,9 +86,9 @@ class ObservationOperations(RelationOperations): _, checksum = await self.write_entity_file(entity) await self.entity_service.update_entity(path_id, {"checksum": checksum}) - # Get final entity with all updates + # Return final entity with all updates return await self.entity_service.get_by_path_id(path_id) except Exception as e: logger.error(f"Failed to delete observations: {e}") - raise + raise \ No newline at end of file diff --git a/src/basic_memory/services/observation_service.py b/src/basic_memory/services/observation_service.py index f9900e16..b5f4495c 100644 --- a/src/basic_memory/services/observation_service.py +++ b/src/basic_memory/services/observation_service.py @@ -1,6 +1,6 @@ """Service for managing observations in the database.""" -from typing import List, Sequence +from typing import List, Sequence, Optional from loguru import logger from sqlalchemy import select @@ -8,6 +8,8 @@ from sqlalchemy import select from basic_memory.models import Observation as ObservationModel from basic_memory.repository.observation_repository import ObservationRepository from .service import BaseService +from ..schemas.base import ObservationCategory +from ..schemas.request import ObservationCreate class ObservationService(BaseService[ObservationRepository]): @@ -20,13 +22,22 @@ class ObservationService(BaseService[ObservationRepository]): super().__init__(observation_repository) async def add_observations( - self, entity_id: int, observations: List[str], context: str | None = None + self, + entity_id: int, + observations: List[str | ObservationCreate], + context: Optional[str] = None, ) -> Sequence[ObservationModel]: """Add multiple observations to an entity.""" logger.debug(f"Adding {len(observations)} observations to entity: {entity_id}") return await self.repository.create_all( [ - dict(entity_id=entity_id, content=observation, context=context) + # unpack the ObservationCreate values if present + dict( + entity_id=entity_id, + content=getattr(observation, "content", observation), + context=context, + category=getattr(observation, "category", None), + ) for observation in observations ] ) @@ -46,14 +57,20 @@ class ObservationService(BaseService[ObservationRepository]): logger.debug(f"Deleting all observations for entity: {entity_id}") return await self.repository.delete_by_fields(entity_id=entity_id) - async def search_observations(self, query: str) -> List[ObservationModel]: + async def search_observations(self, query: str, category: Optional[ObservationCategory] = None) -> List[ObservationModel]: """Search for observations across all entities.""" logger.debug(f"Searching observations with query: {query}") - result = await self.repository.execute_query( - select(ObservationModel).filter( - ObservationModel.content.contains(query) | ObservationModel.context.contains(query) - ) + + # Build base query + statement = select(ObservationModel).filter( + ObservationModel.content.contains(query) | ObservationModel.context.contains(query) ) + + # Add category filter if specified + if category: + statement = statement.filter(ObservationModel.category == category) + + result = await self.repository.execute_query(statement) observations = result.scalars().all() return [ObservationModel(content=obs.content) for obs in observations] @@ -61,3 +78,13 @@ class ObservationService(BaseService[ObservationRepository]): """Get all observations with a specific context.""" logger.debug(f"Getting observations for context: {context}") return await self.repository.find_by_context(context) + + async def get_observations_by_category(self, category: ObservationCategory) -> Sequence[ObservationModel]: + """Get all observations with a specific context.""" + logger.debug(f"Getting observations for context: {category}") + return await self.repository.find_by_category(category) + + async def observation_categories(self) -> Sequence[str]: + """Get all observation categories.""" + logger.debug("Getting observations categories") + return await self.repository.observation_categories() diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.py index 0f8453fa..5f9b7304 100644 --- a/tests/api/test_knowledge_router.py +++ b/tests/api/test_knowledge_router.py @@ -41,7 +41,14 @@ async def create_entity(client) -> EntityResponse: async def add_observations(client, path_id: str) -> List[ObservationResponse]: response = await client.post( "/knowledge/observations", - json={"path_id": path_id, "observations": ["First observation", "Second observation"]}, + json={ + "path_id": path_id, + "observations": [ + {"content": "First observation", "category": "tech"}, + {"content": "Second observation", "category": "note"}, + ], + "context": "something special" + }, ) # Verify observations were added assert response.status_code == 200 @@ -407,9 +414,10 @@ async def test_full_knowledge_flow(client: AsyncClient): json={ "path_id": "test/main_entity", "observations": [ - "Connected to first related entity", - "Connected to second related entity", + {"content": "Connected to first related entity", "category": "tech"}, + {"content": "Connected to second related entity", "category": "note"}, ], + "context": "testing the flow" }, ) @@ -426,7 +434,9 @@ async def test_full_knowledge_flow(client: AsyncClient): # 6. Search should find all related entities search = await client.post("/knowledge/search", json={"query": "Related"}) matches = search.json()["matches"] - assert len(matches) == 3 # Should find both related entities, and the main one with the observation + assert ( + len(matches) == 3 + ) # Should find both related entities, and the main one with the observation # 7. Delete main entity response = await client.post( diff --git a/tests/markdown/test_knowledge_writer.py b/tests/markdown/test_knowledge_writer.py index ae10afc0..6075fbb2 100644 --- a/tests/markdown/test_knowledge_writer.py +++ b/tests/markdown/test_knowledge_writer.py @@ -1,9 +1,14 @@ """Tests for knowledge entity writer.""" +from datetime import datetime, UTC import pytest from basic_memory.markdown.knowledge_writer import KnowledgeWriter -from basic_memory.models import Entity as EntityModel, Observation, Relation +from basic_memory.models import ( + Entity as EntityModel, + Observation, + Relation, ObservationCategory, +) @pytest.fixture @@ -15,17 +20,40 @@ def writer(): @pytest.fixture def test_entity(): """Create test entity with observations and relations.""" + now = datetime.now(UTC) # Create main entity - entity = EntityModel(id=123, name="TestEntity", entity_type="test", description="A test entity") + entity = EntityModel( + id=1, + path_id="test/test_entity", + name="TestEntity", + entity_type="test", + description="A test entity", + created_at=now, + updated_at=now + ) - # Add observations + # Add observations with categories and context entity.observations = [ - Observation(content="First observation"), - Observation(content="Second observation"), + Observation( + content="Technical implementation detail", + category=ObservationCategory.TECH.value, + context="Initial implementation" + ), + Observation( + content="Design pattern choice", + category=ObservationCategory.DESIGN.value + ), ] # Create related entity - other_entity = EntityModel(id=456, name="OtherEntity", entity_type="test") + other_entity = EntityModel( + id=2, + path_id="test/other_entity", + name="OtherEntity", + entity_type="test", + created_at=now, + updated_at=now + ) # Create relation from main entity to other relation = Relation(from_entity=entity, to_entity=other_entity, relation_type="relates_to") @@ -40,28 +68,78 @@ async def test_format_frontmatter(writer: KnowledgeWriter, test_entity: EntityMo frontmatter = await writer.format_frontmatter(test_entity) assert frontmatter["type"] == "test" - assert frontmatter["id"] == 123 + assert frontmatter["id"] == "test/test_entity" assert isinstance(frontmatter["created"], str) assert isinstance(frontmatter["modified"], str) @pytest.mark.asyncio -async def test_format_content_basic(writer: KnowledgeWriter, test_entity: EntityModel): - """Test basic content formatting without metadata.""" +async def test_format_content_with_categories(writer: KnowledgeWriter, test_entity: EntityModel): + """Test content formatting with categorized observations.""" content = await writer.format_content(test_entity) - # Check sections - assert content.startswith("# TestEntity\n") - assert "A test entity" in content + # Check observations section header and format comment assert "## Observations" in content - assert "- First observation" in content - assert "- Second observation" in content - assert "## Relations" in content - assert "- [[OtherEntity]] relates_to" in content + assert "" in content - # Should not have metadata section - assert "# Metadata" not in content - assert "```yml" not in content + # Check formatted observations + assert "- [tech] Technical implementation detail (Initial implementation)" in content + assert "- [design] Design pattern choice" in content + + +@pytest.mark.asyncio +async def test_format_content_default_category(writer: KnowledgeWriter): + """Test formatting observation with default category.""" + entity = EntityModel(id=1, name="Test", entity_type="test") + entity.observations = [ + Observation(content="Simple note", category=ObservationCategory.NOTE.value) + ] + + content = await writer.format_content(entity) + assert "- [note] Simple note" in content + + +@pytest.mark.asyncio +async def test_format_content_context_handling(writer: KnowledgeWriter): + """Test formatting observations with different context scenarios.""" + entity = EntityModel(id=1, name="Test", entity_type="test") + entity.observations = [ + # With context + Observation( + content="With context", + category=ObservationCategory.TECH.value, + context="Important context" + ), + # Without context + Observation( + content="No context", + category=ObservationCategory.TECH.value + ), + ] + + content = await writer.format_content(entity) + assert "- [tech] With context (Important context)" in content + assert "- [tech] No context" in content + assert "No context ()" not in content # Shouldn't have empty parentheses + + +@pytest.mark.asyncio +async def test_format_content_sections_order(writer: KnowledgeWriter, test_entity: EntityModel): + """Test proper order and spacing of sections with new format.""" + content = await writer.format_content(test_entity) + lines = content.split("\n") + + # Find key sections + title_idx = next(i for i, line in enumerate(lines) if line.startswith("# ")) + obs_idx = next(i for i, line in enumerate(lines) if line.strip() == "## Observations") + format_idx = next(i for i, line in enumerate(lines) if "