From 931a5f31c584e93097830c94b72098f99ccb90c8 Mon Sep 17 00:00:00 2001 From: phernandez Date: Sat, 7 Dec 2024 09:54:49 -0600 Subject: [PATCH] refactoring MemoryService, adding stored memory.json --- memory.json | 140 +++++++- src/basic_memory/schemas.py | 54 +++ src/basic_memory/services.py | 329 ++++++++++++++++++ src/basic_memory/services/entity_service.py | 107 ++---- src/basic_memory/services/memory_service.py | 120 ++----- .../services/observation_service.py | 102 ++---- src/basic_memory/services/relation_service.py | 124 ++----- 7 files changed, 636 insertions(+), 340 deletions(-) create mode 100644 src/basic_memory/services.py diff --git a/memory.json b/memory.json index 5d3d2e0a..f5ba1c1c 100644 --- a/memory.json +++ b/memory.json @@ -56,6 +56,56 @@ {"type":"entity","name":"Basic_Memory_Schema_Design","entityType":"Implementation","observations":["Uses Pydantic for domain models and validation","Automatic ID generation with timestamp and UUID","Clear separation from SQLAlchemy storage models","Supports optional context tracking","Models match markdown file structure","Enables clean serialization/deserialization","Strong typing with proper validation rules","Independent from storage concerns"]} {"type":"entity","name":"Basic_Memory_Next_Tasks","entityType":"TaskList","observations":["✅ Implement SQLAlchemy models and repositories (Done)","✅ Add SQLAlchemy migrations (Done)","✅ Create service layer (Done)","✅ Implement file I/O module (Done)","✅ Set up domain models with Pydantic (Done)","✅ Initial test infrastructure (Done)","✅ Basic CRUD operations (Done)","⏳ Implement full test coverage for db.py","⏳ Add more sophisticated search functionality","⏳ Implement CLI interface","⏳ Add relationship management to services","⏳ Handle concurrent file operations safely","⏳ Add versioning for file changes","⏳ Implement proper backup strategy","⏳ Add type hints throughout codebase","⏳ Improve error messages and logging","⏳ Add documentation for core modules"]} {"type":"entity","name":"Basic_Memory_Meta_Experience","entityType":"Case_Study","observations":["Experienced our own context loss when reconstructing project knowledge","Had to rebuild task list and project context from filesystem and memory","Validated 'filesystem as source of truth' principle through reconstruction","Code and tests served as reliable historical record","Knowledge graph structure helped guide reconstruction process","Markdown files provided human-readable context","Atomic information design made piece-by-piece reconstruction possible","Ironic validation of the need for basic-memory's features","Experience demonstrates value of durable, human-readable knowledge storage","Shows importance of separating durable storage from ephemeral context"]} +{"type":"entity","name":"Model_Context_Protocol","entityType":"protocol","observations":["Core part of the basic-memory architecture","Enables AI-human collaboration on projects","Provides tool-based interaction with knowledge graph","Developed by Anthropic for structured AI-system interaction","Used for maintaining consistent, rich context across conversations"]} +{"type":"entity","name":"basic-memory_core_principles","entityType":"principles","observations":["Local First: All data stored locally in SQLite","Project Isolation: Separate databases per project","Human Readable: Everything exportable to plain text","AI Friendly: Structure optimized for LLM interaction","DIY Ethics: User owns and controls their data","Simple Core: Start simple, expand based on needs","Tool Integration: MCP-based interaction model"]} +{"type":"entity","name":"basic-memory_business_model","entityType":"business_strategy","observations":["Core features free: Local SQLite, basic knowledge graph, search, markdown export, basic MCP tools","Professional features potential: Rich document export, advanced versioning, collaboration features, custom integrations, priority support","Focus on maintaining DIY/punk philosophy while enabling sustainability"]} +{"type":"entity","name":"basic-memory_cli","entityType":"interface","observations":["Supports project management commands (create, switch, list)","Entity management (add entity, add observation, add relation)","Future support for export and batch operations","Follows consistent command structure","Planned integration with MCP tools"]} +{"type":"entity","name":"basic-memory_export_format","entityType":"file_format","observations":["Uses markdown with frontmatter metadata","Includes entity name, type, creation timestamp","Observations as bullet points","Relations in structured format with links","References section at bottom","Designed for human readability and machine parsing","Example format documented in project specs"]} +{"type":"entity","name":"relation_service","entityType":"code_module","observations":["Planned service for managing relations in both filesystem and database","Will follow filesystem-is-source-of-truth principle like other services","Needs to handle atomic file operations for relation updates","Must coordinate with EntityService for relationship integrity","Will handle bidirectional relationship tracking","Will support relation validation and type enforcement","Must implement rebuild functionality for index recovery","Will need careful error handling for file/db sync","Should support relation search and filtering","Must handle relation lifecycle (create/read/update/delete)"]} +{"type":"entity","name":"service_layer_patterns","entityType":"implementation_patterns","observations":["Services handle both file and database operations","Filesystem is always source of truth","Database serves as queryable index","Services implement atomic file operations","Clear error hierarchy with specific exceptions","Use of dependency injection via constructor params","Async/await used throughout service layer","Services coordinate between storage layers","Repository pattern used for database access","Services maintain entity integrity across storage","Rich error types extend from ServiceError base","Rebuild operations available for recovery"]} +{"type":"entity","name":"database_models","entityType":"implementation","observations":["Entity model with unique name and type","Observation model linked to entities","Relation model tracks connections between entities","Custom UTCDateTime type for timestamp handling","Use of SQLAlchemy relationships for navigation","Cascading deletes for dependent objects","String IDs used for compatibility","Rich relationship modeling with backpopulates","Proper indexing on foreign keys","Context tracking available on models","Models include created_at timestamps","Relationships handle bidirectional navigation"]} +{"type":"entity","name":"repository_patterns","entityType":"implementation_patterns","observations":["Generic Repository[T] base class implementation","Type-safe operations with SQLAlchemy","Specialized repositories for each model type","Async operations throughout","Clear error handling patterns","Support for custom queries and filtering","Pagination support built-in","Transaction management via session","Proper type hints and generics usage","Entity-specific query methods in subclasses"]} +{"type":"entity","name":"relation_service_design","entityType":"design","observations":["Must handle relation lifecycle in both files and DB","Needs to validate existence of both entities","Should support relation type enforcement","Must maintain bidirectional consistency","Should support relation querying and filtering","Needs proper error handling for graph consistency","Must integrate with entity file format","Should support bulk operations for efficiency","Must handle relation deletion and cascading","Should provide search by type and entities"]} +{"type":"entity","name":"relation_service_implementation_plan","entityType":"plan","observations":["1. Define core relation operations (create, get, delete)","2. Implement file format handling for relations","3. Add database sync with RelationRepository","4. Implement validation and error handling","5. Add rebuild and recovery operations","6. Implement relation type enforcement","7. Add relation search and filtering","8. Implement bulk operations","9. Add comprehensive tests","10. Document API and error handling"]} +{"type":"entity","name":"relation_service_challenges","entityType":"challenges","observations":["Maintaining consistency between file and database","Handling relation type validation efficiently","Managing bidirectional relationships in files","Ensuring atomic updates across entities","Handling deletion with proper cascading","Efficient querying of relation graphs","Recovery from partial file/db sync failures","Bulk operation atomicity","Clear error reporting for graph operations","Performance with large relation sets"]} +{"type":"entity","name":"relation_file_format","entityType":"file_format","observations":["Relations stored in entity markdown files","Format: [target_id] relation_type | context","Relations section marked by ## Relations header","Outgoing relations only stored in source entity","Relations rebuild on entity load","Clean human-readable format","Context is optional with pipe separator","Links generate valid navigation references","Markdown-friendly formatting","Example: [Paul] authored | with Claude"]} +{"type":"entity","name":"relation_service_error_handling","entityType":"implementation_patterns","observations":["RelationError extends ServiceError base","Specific errors for validation failures","Handles entity not found cases","Manages relation type validation errors","File operation errors properly wrapped","Database sync errors clearly reported","Transaction rollback on errors","Proper error propagation chain","Clear error messages for debugging","Recovery paths for common errors"]} +{"type":"entity","name":"relation_service_testing","entityType":"testing","observations":["Test all relation lifecycle operations","Verify file and database consistency","Test relation type validation","Check error handling paths","Test bulk operations","Verify bidirectional consistency","Test recovery operations","Check cascade operations","Verify search and filtering","Test with large relation sets"]} +{"type":"entity","name":"fileio_patterns","entityType":"implementation_patterns","observations":["Atomic file operations with temporary files","Clear error handling for IO operations","Consistent file naming and paths","Support for different file formats","Efficient file reading and writing","Proper file locking mechanisms","Recovery from partial writes","Consistent encoding handling","Directory management utilities","Path manipulation helpers","Currently implemented in fileio.py module","Uses pathlib for path operations","Handles file not found cases gracefully","Maintains data integrity during writes"]} +{"type":"entity","name":"pytest_patterns","entityType":"implementation_patterns","observations":["Common fixtures should be in conftest.py for reuse","Use pytest_asyncio.fixture for async fixtures","Session fixtures need proper async cleanup","Temporary directories should be managed with context managers","Test categories: happy path, error path, recovery, edge cases","Services need project_path and repo injected","Use monkeypatch for mocking in async context","SQLite in-memory database ideal for testing","Explicit test verification: file content and database state"]} +{"type":"entity","name":"relation_implementation_learnings","entityType":"implementation_learnings","observations":["Better to pass full Entity objects than IDs to services","Services should not re-read entities if they have them","File operations should be atomic and verified","Database serves as queryable index, not source of truth","Relations stored in source entity's markdown file","Clear separation between file ops and database sync","Entity objects should own their relations list","Context is optional but fully supported in implementation"]} +{"type":"entity","name":"test_driven_insights","entityType":"learnings","observations":["Tests help reveal better API design (e.g., passing Entity objects)","Error cases drive proper exception hierarchy","File verification as important as database checks","Edge cases inform markdown format decisions","Recovery tests ensure system resilience","Tests document expected behavior clearly","Fixtures significantly reduce test complexity","Common patterns emerge through test writing"]} +{"type":"entity","name":"meta_development_insights","entityType":"process","observations":["Break down large tasks into reviewable chunks","One file at a time prevents response truncation","Iterative development with tests leads to better design","Infrastructure code (fixtures) should be consolidated early","Test categories help ensure comprehensive coverage","Knowledge capture should happen during development","APIs tend to evolve toward simpler patterns","File operations require careful verification"]} +{"type":"entity","name":"AI_Assistant_Learnings","entityType":"meta_insights","observations":["Output management: Breaking responses into single files prevents truncation and allows better review","Knowledge graph helps maintain context: I can reference previous decisions and patterns accurately","Memory rebuilding experience validated the need for durable storage","Test-driven development provides clear steps and verification","Explicit relation tracking in knowledge graph helps me understand project context","Rich context from multiple sources (code, docs, tests) enables better assistance","File-at-a-time approach allows deeper analysis of each component","Keeping entity names consistent helps with referencing and relationships"]} +{"type":"entity","name":"Effective_Response_Patterns","entityType":"meta_patterns","observations":["When showing code changes, break into discrete files","Review existing code before suggesting changes","Reference knowledge graph for context and patterns","Explicitly connect new code to existing patterns","Validate suggestions against test cases","Keep track of file changes for atomic commits","Check both implementation and test files for consistency","Maintain clear separation of concerns in responses"]} +{"type":"entity","name":"AI_Context_Management","entityType":"meta_practice","observations":["Knowledge graph provides reliable persistent memory","Project documentation gives high-level context","Code review shows implementation patterns","Tests demonstrate expected behavior","Important to actively track what has been modified","Entity relationships help understand dependencies","Regular knowledge capture during development","Using consistent entity references across conversations"]} +{"type":"entity","name":"AI_Tool_Usage_Patterns","entityType":"meta_practice","observations":["read_file before suggesting changes","write_file one file at a time","list_directory to understand project structure","search_nodes to find relevant context","create_entities to capture new learnings","create_relations to connect concepts","Using knowledge graph to track decisions","Validating changes through test execution"]} +{"type":"entity","name":"relation_service_learnings","entityType":"implementation_learnings","observations":["Entity-based API cleaner than ID-based for service layer","Model_dump method can handle storage serialization","File format needs explicit section markers (## Relations)","Whitespace handling important for long content comparisons","Test fixtures allow focused test cases","SQLAlchemy selects better than raw SQL for type safety","Atomic file operations maintained for relations"]} +{"type":"entity","name":"test_driven_insights_relations","entityType":"learnings","observations":["Tests revealed need for whitespace normalization","Edge cases drove file format decisions","SQLAlchemy model access safer than raw queries","Fixtures reduced test setup complexity","File verification as important as database checks","Testing both memory model and storage format","Test categories ensure comprehensive coverage"]} +{"type":"entity","name":"relation_service_patterns","entityType":"patterns","observations":["Use Entity objects in API","Serialize to IDs for storage","Maintain file as source of truth","Keep file format human-readable","Handle circular references in serialization","Use repository pattern for database","Clear error hierarchies"]} +{"type":"entity","name":"packaging_learnings","entityType":"technical_learnings","observations":["When using pytest-mock, traditional pip install works more reliably than uv sync","Package discovery behavior can differ between uv and pip","Clean venv with pip install is a reliable fallback for dependency issues","Package installation location might differ between uv and pip","Dependencies in pyproject.toml dev section work reliably with pip install -e .[dev]"]} +{"type":"entity","name":"Recent_Implementation_Progress","entityType":"progress_update","observations":["Successfully split services.py into modular structure under services/","Created __init__.py, entity_service.py, observation_service.py, relation_service.py","Fixed pytest-mock installation issues by using pip install -e .[dev] instead of uv sync","Improved test structure with minimal mocking - only used for error testing","Implemented relation service with Entity-based API","Achieved good test coverage across services","File operations are only mocked when testing error conditions","Services follow filesystem-as-source-of-truth pattern"]} +{"type":"entity","name":"Next_Steps","entityType":"project_tasks","observations":["Consider adding more relation service tests","Potentially expand relations features","Look for opportunities to improve test coverage","Consider documenting package management preferences (pip vs uv)","Consider adding integration tests for services","Review and possibly expand error handling cases"]} +{"type":"entity","name":"Development_Practices","entityType":"process","observations":["Favor real operations over mocks in tests","Only mock for error condition testing","Use pip install -e .[dev] for reliable dev dependency installation","Maintain modular service structure","Keep filesystem as source of truth","Use Entity objects in service APIs instead of IDs","Validate both file and database state in tests"]} +{"type":"entity","name":"MCP_Resources","entityType":"Concept","observations":["Stateful objects in Model Context Protocol","Enable persistent access to capabilities"]} +{"type":"entity","name":"MCP_Server_Implementation","entityType":"Technical_Design","observations":["Inherits from mcp.server.Server base class","Tools are implemented as async methods","Each tool method maps directly to a function available to the AI","Tools can request user input via Prompts","Simple function call interface rather than explicit resource management","State management handled by server instance","Returns serialized data using model_dump() for consistency"]} +{"type":"entity","name":"MCP_Tools","entityType":"Protocol_Feature","observations":["Defined as async methods on server class","Return values must match tool definition schema","Can maintain state between invocations via server instance","Tools can prompt for user input when needed","No need for explicit Resource objects in implementation"]} +{"type":"entity","name":"Basic_Memory_MCP","entityType":"Implementation","observations":["Uses MemoryService for core operations","Implements project selection via prompts","Maintains project context across tool invocations","Maps directly to memory graph operations","Handles serialization of Pydantic models"]} +{"type":"entity","name":"Basic_Memory_Testing","entityType":"Testing_Design","observations":["Needs pytest for async testing","Should isolate filesystem operations for tests","Needs to handle MCP server lifecycle in tests","Should test both service layer and MCP interface","Will need mocks for project paths and file operations"]} +{"type":"entity","name":"Memory_Service_Tests","entityType":"Test_Suite","observations":["Should test entity creation with observations","Should test relation creation between entities","Should verify proper ID generation and model validation","Should test deletion cascading","Should test search functionality","Must verify proper serialization of entities and relations"]} +{"type":"entity","name":"MCP_Server_Tests","entityType":"Test_Suite","observations":["Should test project initialization workflow","Should test prompt handling","Should verify tool input/output formats","Should test error cases and validation","Must verify proper serialization in tool responses"]} +{"type":"entity","name":"Memory_Service_Refactoring","entityType":"Technical_Task","observations":["MemoryService uses create() but EntityService might expect create_entity()","MemoryService assumes get_by_name() but EntityService might use different method","Need to verify deletion method signatures","Need to check if search interface matches","Should verify observation handling matches ObservationService interface","RelationService methods need verification","EntityService.create_entity takes name, type, and optional observations directly, not an Entity object","EntityService requires project_path and entity_repo in constructor","ObservationService.add_observation takes Entity object and content string, not raw data","RelationService.create_relation takes Entity objects directly, not dict data","All services follow filesystem-as-source-of-truth pattern with DB indexing","All services handle database synchronization internally","Services expect Path objects for filesystem operations"]} +{"type":"entity","name":"Service_Interface_Audit","entityType":"Technical_Task","observations":["Need to review all existing service interfaces","Document current method signatures","Map discrepancies between MemoryService assumptions and actual interfaces","Check return types and error handling patterns","Review transaction/atomicity requirements","Method signatures need alignment: create vs create_entity etc","Need to handle DB repositories in service constructors","File operations should use project_path consistently","Need to maintain filesystem-as-source-of-truth pattern","Should handle database synchronization at service level","Error handling should align with existing patterns","Consider making MemoryService handle DB indexing consistently"]} +{"type":"entity","name":"Memory_Service_Patterns","entityType":"Technical_Pattern","observations":["Uses inner async functions to encapsulate operation logic","Leverages list comprehensions with async functions for parallel operations","Each operation follows a consistent pattern: validate, update DB, write file","Inner functions make the code more readable and maintainable","Operations can run in parallel when using list comprehensions with async functions"]} +{"type":"entity","name":"Pydantic_Create_Pattern","entityType":"Technical_Pattern","observations":["Separate Create models match the exact shape of incoming data","Provides clear contract for MCP tool inputs","Handles validation of raw input data","Converts cleanly to domain models via from_create methods","Maintains separation between external API format and internal models","Similar to FastAPI request model pattern","Allows camelCase in API while using snake_case internally"]} +{"type":"entity","name":"Basic_Memory_Business","entityType":"Business_Model","observations":["Core system is open source and free","Local-first, giving users data control","Professional features could be licensed","Enterprise support and customization services","Potential for MCP tool marketplace"]} +{"type":"entity","name":"MCP_Marketplace","entityType":"Business_Concept","observations":["Could host verified MCP tools for different use cases","Tools rated by performance and reliability","Marketplace takes percentage of tool usage fees","Enterprise tool verification and security scanning","Custom tool development services","Integration support for existing tools"]} +{"type":"entity","name":"Persistence_Of_Vision","entityType":"Concept","observations":["Mental model for continuous AI-human interaction","Like cinema: 24fps creates illusion of smooth motion","Basic-memory provides 'frames' of structured knowledge","Current state: Better than flipbook, not yet digital cinema","Goal: Achieve smoother cognitive continuity between interactions","Proposed by Drew as metaphor for AI conversation continuity"]} +{"type":"entity","name":"Conversation_Continuity_Pattern","entityType":"Usage_Pattern","observations":["Use basic-memory entity/relation schema for conversations","Each chat becomes an entity with observations for key points","Relations link to discussed concepts and other chats","Uses zettelkasten format IDs for natural ordering","Can be used as template/recipe for others","Future possibility: Git SHA integration for versioning"]} +{"type":"entity","name":"Usage_Recipes","entityType":"Feature_Concept","observations":["Predefined patterns users can follow or adapt","Could include conversation tracking recipe","Templates for different knowledge management styles","Shows practical applications of the generic schema","Helps users get started with the system"]} +{"type":"entity","name":"Chat_References","entityType":"Technical_Feature","observations":["Uses ref:* syntax to reference previous conversations","Combines reference semantics with pointer symbolism","Format: ref:*{zettelkasten-id}","Allows explicit context loading between chats","Inspired by C++ references and pointers","Provides memory-model-like access to conversation context","Uses ref:// URI format following MCP Resource pattern","Could support multiple reference schemes (chat/entity/concept)","Makes reference semantics explicit and unambiguous","Aligns with standard URI formatting"]} +{"type":"entity","name":"Chat_Reference_Protocol","entityType":"Technical_Specification","observations":["Uses URI format: ref://basic-memory/chat/[id]","Follows MCP Resource pattern: [protocol]://[host]/[path]","Enables explicit context loading between chats","Can support multiple resource types (chat/entity/concept)","Provides standardized way to reference previous conversations","Example: ref://basic-memory/chat/20240307-drew-ab12ef34"]} +{"type":"entity","name":"20240307-chat-reference-protocol","entityType":"conversation","observations":["Developed ref:// URI format for chat references","Added Chat Reference Protocol to prompt instructions","Discussed implementation of chat continuation","Created complete prompt instructions document","Reference format follows MCP Resource pattern","Reviewed and confirmed complete prompt instructions","Ready to test ref://basic-memory/chat/20240307-chat-reference-protocol in new chat"]} {"type":"relation","from":"Paul","to":"Basic_Machines","relationType":"created_and_maintains"} {"type":"relation","from":"basic-memory","to":"Basic_Machines","relationType":"is_component_of"} {"type":"relation","from":"Paul","to":"basic-memory","relationType":"develops"} @@ -202,4 +252,92 @@ {"type":"relation","from":"Basic_Memory_Meta_Experience","to":"basic-memory","relationType":"validates_design_of"} {"type":"relation","from":"Basic_Memory_Meta_Experience","to":"DIY_Ethics","relationType":"demonstrates_principles_of"} {"type":"relation","from":"Basic_Memory_Meta_Experience","to":"design_decisions","relationType":"reinforces"} -{"type":"relation","from":"Basic_Memory_Meta_Experience","to":"Basic_Memory_Current_Architecture","relationType":"validates"} \ No newline at end of file +{"type":"relation","from":"Basic_Memory_Meta_Experience","to":"Basic_Memory_Current_Architecture","relationType":"validates"} +{"type":"relation","from":"Model_Context_Protocol","to":"basic-memory","relationType":"enables"} +{"type":"relation","from":"basic-memory_core_principles","to":"basic-memory","relationType":"guides"} +{"type":"relation","from":"basic-memory_core_principles","to":"DIY_Ethics","relationType":"aligns_with"} +{"type":"relation","from":"basic-memory_business_model","to":"basic-memory","relationType":"defines_sustainability_for"} +{"type":"relation","from":"basic-memory_business_model","to":"DIY_Ethics","relationType":"maintains_alignment_with"} +{"type":"relation","from":"basic-memory_cli","to":"basic-memory","relationType":"provides_interface_for"} +{"type":"relation","from":"basic-memory_cli","to":"Model_Context_Protocol","relationType":"integrates_with"} +{"type":"relation","from":"basic-memory_export_format","to":"basic-memory","relationType":"standardizes_output_of"} +{"type":"relation","from":"basic-memory_export_format","to":"markdown_format","relationType":"extends"} +{"type":"relation","from":"basic-memory_core_principles","to":"Basic_Machines_Philosophy","relationType":"implements"} +{"type":"relation","from":"Model_Context_Protocol","to":"AI_Human_Collaboration_Model","relationType":"enables"} +{"type":"relation","from":"relation_service","to":"basic-memory","relationType":"will_be_component_of"} +{"type":"relation","from":"relation_service","to":"service_layer_patterns","relationType":"follows"} +{"type":"relation","from":"relation_service","to":"fileio_patterns","relationType":"uses"} +{"type":"relation","from":"relation_service","to":"database_models","relationType":"uses"} +{"type":"relation","from":"relation_service","to":"repository_patterns","relationType":"implements"} +{"type":"relation","from":"relation_service_design","to":"relation_service","relationType":"guides_implementation_of"} +{"type":"relation","from":"relation_service_implementation_plan","to":"relation_service","relationType":"defines_implementation_of"} +{"type":"relation","from":"relation_service_challenges","to":"relation_service_design","relationType":"informs"} +{"type":"relation","from":"relation_file_format","to":"markdown_format","relationType":"extends"} +{"type":"relation","from":"relation_service_error_handling","to":"service_layer_patterns","relationType":"implements"} +{"type":"relation","from":"relation_service_testing","to":"testing_infrastructure","relationType":"extends"} +{"type":"relation","from":"fileio_patterns","to":"service_layer_patterns","relationType":"enables"} +{"type":"relation","from":"database_models","to":"repository_patterns","relationType":"enables"} +{"type":"relation","from":"relation_service","to":"entity_service","relationType":"coordinates_with"} +{"type":"relation","from":"relation_file_format","to":"relation_service","relationType":"defines_storage_for"} +{"type":"relation","from":"relation_service_error_handling","to":"relation_service","relationType":"ensures_reliability_of"} +{"type":"relation","from":"relation_service_testing","to":"relation_service","relationType":"verifies"} +{"type":"relation","from":"service_layer_patterns","to":"basic-memory_implementation_patterns","relationType":"implements"} +{"type":"relation","from":"repository_patterns","to":"basic-memory_implementation_patterns","relationType":"implements"} +{"type":"relation","from":"fileio_patterns","to":"basic-memory_implementation_patterns","relationType":"implements"} +{"type":"relation","from":"database_models","to":"basic-memory_implementation_patterns","relationType":"implements"} +{"type":"relation","from":"relation_service_challenges","to":"implementation_challenges","relationType":"extends"} +{"type":"relation","from":"relation_service_implementation_plan","to":"future_work","relationType":"details"} +{"type":"relation","from":"relation_service_design","to":"design_decisions","relationType":"aligns_with"} +{"type":"relation","from":"relation_file_format","to":"design_decisions","relationType":"follows"} +{"type":"relation","from":"pytest_patterns","to":"testing_infrastructure","relationType":"extends"} +{"type":"relation","from":"relation_implementation_learnings","to":"basic-memory_implementation_patterns","relationType":"informs"} +{"type":"relation","from":"test_driven_insights","to":"test_driven_development","relationType":"enriches"} +{"type":"relation","from":"meta_development_insights","to":"AI_Human_Collaboration_Model","relationType":"improves"} +{"type":"relation","from":"relation_implementation_learnings","to":"relation_service","relationType":"guides_implementation_of"} +{"type":"relation","from":"pytest_patterns","to":"test_evolution","relationType":"demonstrates"} +{"type":"relation","from":"test_driven_insights","to":"design_decisions","relationType":"influences"} +{"type":"relation","from":"meta_development_insights","to":"architecture_evolution","relationType":"informs"} +{"type":"relation","from":"relation_service","to":"relation_implementation_learnings","relationType":"validates"} +{"type":"relation","from":"test_driven_insights","to":"implementation_challenges","relationType":"helps_solve"} +{"type":"relation","from":"AI_Assistant_Learnings","to":"meta_development_insights","relationType":"enriches"} +{"type":"relation","from":"Effective_Response_Patterns","to":"AI_Assistant_Learnings","relationType":"implements"} +{"type":"relation","from":"AI_Context_Management","to":"AI_Human_Collaboration_Model","relationType":"improves"} +{"type":"relation","from":"AI_Tool_Usage_Patterns","to":"AI_Context_Management","relationType":"enables"} +{"type":"relation","from":"AI_Assistant_Learnings","to":"Basic_Memory_Meta_Experience","relationType":"validates"} +{"type":"relation","from":"AI_Tool_Usage_Patterns","to":"Model_Context_Protocol","relationType":"demonstrates_effective_use_of"} +{"type":"relation","from":"AI_Context_Management","to":"basic-memory","relationType":"validates_design_of"} +{"type":"relation","from":"Effective_Response_Patterns","to":"AI_Human_Development_Methodology","relationType":"refines"} +{"type":"relation","to":"relation_service","from":"relation_service_patterns","relationType":"guides"} +{"type":"relation","to":"test_driven_development","from":"test_driven_insights_relations","relationType":"enriches"} +{"type":"relation","to":"implementation_challenges","from":"relation_service_learnings","relationType":"solves"} +{"type":"relation","to":"basic-memory_implementation_patterns","from":"relation_service_patterns","relationType":"implements"} +{"type":"relation","to":"markdown_format","from":"relation_service_patterns","relationType":"extends"} +{"type":"relation","to":"service_layer_patterns","from":"relation_service_patterns","relationType":"refines"} +{"type":"relation","from":"packaging_learnings","to":"implementation_challenges","relationType":"informs"} +{"type":"relation","from":"packaging_learnings","to":"test_driven_development","relationType":"impacts"} +{"type":"relation","to":"basic-memory","from":"Recent_Implementation_Progress","relationType":"updates_status_of"} +{"type":"relation","to":"future_work","from":"Next_Steps","relationType":"extends"} +{"type":"relation","to":"design_decisions","from":"Development_Practices","relationType":"informs"} +{"type":"relation","to":"packaging_learnings","from":"Development_Practices","relationType":"incorporates"} +{"type":"relation","to":"test_driven_development","from":"Development_Practices","relationType":"refines"} +{"type":"relation","to":"basic-memory_implementation_patterns","from":"Development_Practices","relationType":"enhances"} +{"type":"relation","from":"Basic_Memory_MCP","to":"MCP_Server_Implementation","relationType":"follows"} +{"type":"relation","from":"Basic_Memory_MCP","to":"MCP_Tools","relationType":"uses"} +{"type":"relation","from":"Basic_Memory","to":"MCP_Server_Implementation","relationType":"implements"} +{"type":"relation","from":"Basic_Memory_Testing","to":"Memory_Service_Tests","relationType":"includes"} +{"type":"relation","from":"Basic_Memory_Testing","to":"MCP_Server_Tests","relationType":"includes"} +{"type":"relation","from":"Memory_Service_Tests","to":"Basic_Memory_MCP","relationType":"validates"} +{"type":"relation","from":"MCP_Server_Tests","to":"Basic_Memory_MCP","relationType":"validates"} +{"type":"relation","from":"Service_Interface_Audit","to":"Memory_Service_Refactoring","relationType":"informs"} +{"type":"relation","from":"Memory_Service_Refactoring","to":"Basic_Memory_MCP","relationType":"affects"} +{"type":"relation","from":"Memory_Service_Patterns","to":"Basic_Memory_MCP","relationType":"improves"} +{"type":"relation","from":"Pydantic_Create_Pattern","to":"Memory_Service_Patterns","relationType":"enables"} +{"type":"relation","from":"Pydantic_Create_Pattern","to":"Basic_Memory_MCP","relationType":"improves"} +{"type":"relation","from":"MCP_Marketplace","to":"Basic_Memory_Business","relationType":"enables"} +{"type":"relation","from":"Basic_Memory","to":"MCP_Marketplace","relationType":"could_integrate_with"} +{"type":"relation","from":"Persistence_Of_Vision","to":"Basic_Memory","relationType":"helps_achieve"} +{"type":"relation","from":"Drew","to":"Persistence_Of_Vision","relationType":"conceptualized"} +{"type":"relation","to":"Usage_Recipes","from":"Conversation_Continuity_Pattern","relationType":"is_example_of"} +{"type":"relation","to":"Basic_Memory","from":"Usage_Recipes","relationType":"enhances"} +{"type":"relation","to":"Basic_Memory","from":"Chat_References","relationType":"enhances"} +{"type":"relation","to":"Conversation_Continuity_Pattern","from":"Chat_References","relationType":"implements"} \ No newline at end of file diff --git a/src/basic_memory/schemas.py b/src/basic_memory/schemas.py index 0e1c187d..5aa79393 100644 --- a/src/basic_memory/schemas.py +++ b/src/basic_memory/schemas.py @@ -15,6 +15,41 @@ class Observation(BaseModel): content: str +class ObservationCreate(BaseModel): + """Schema for creating a new observation.""" + content: str + + +class EntityCreate(BaseModel): + """Schema for creating a new entity via the MCP tool interface.""" + name: str + entityType: str # Matches the JSON field name from MCP tool + observations: Optional[List[str]] = None + + +class RelationCreate(BaseModel): + """Schema for creating a new relation via the MCP tool interface.""" + from_: str = None # Raw entity name from MCP tool + to: str + relationType: str + + # Handle the 'from' field which is a Python keyword + @model_validator(mode='before') + @classmethod + def handle_from_field(cls, data: dict) -> dict: + """Convert 'from' to 'from_' if present""" + if 'from' in data: + data['from_'] = data.pop('from') + return data + + def model_dump(self, **kwargs) -> Dict[str, Any]: + """Convert back to format with 'from' field""" + data = super().model_dump(**kwargs) + if 'from_' in data: + data['from'] = data.pop('from_') + return data + + class Relation(BaseModel): """ Represents a directed edge between entities in the knowledge graph. @@ -44,6 +79,15 @@ class Relation(BaseModel): 'context': self.context } + @classmethod + def from_create(cls, create_data: RelationCreate, from_entity: 'Entity', to_entity: 'Entity') -> 'Relation': + """Create a Relation from a RelationCreate schema and actual entities.""" + return cls( + from_entity=from_entity, + to_entity=to_entity, + relation_type=create_data.relationType + ) + class Entity(BaseModel): """ @@ -67,6 +111,16 @@ class Entity(BaseModel): data['id'] = f"{timestamp}-{normalized_name}-{uuid4().hex[:8]}" return data + @classmethod + def from_create(cls, data: EntityCreate) -> 'Entity': + """Create an Entity from an EntityCreate schema.""" + observations = [Observation(content=obs) for obs in (data.observations or [])] + return cls( + name=data.name, + entity_type=data.entityType, + observations=observations + ) + def model_dump(self, **kwargs) -> Dict[str, Any]: """Serialize entity, handling relations to prevent circular references""" # Get basic data without relations diff --git a/src/basic_memory/services.py b/src/basic_memory/services.py new file mode 100644 index 00000000..04e912b8 --- /dev/null +++ b/src/basic_memory/services.py @@ -0,0 +1,329 @@ +from datetime import datetime, UTC +from pathlib import Path +from typing import Optional, List +from uuid import uuid4 +from sqlalchemy import and_, select, delete + +from basic_memory.models import Entity as DbEntity # Rename to avoid confusion +from basic_memory.models import Observation as DbObservation +from basic_memory.models import Relation as DbRelation +from basic_memory.repository import EntityRepository, ObservationRepository, RelationRepository +from basic_memory.schemas import Entity, Observation, Relation +from basic_memory.fileio import ( + read_entity_file, write_entity_file, delete_entity_file, + FileOperationError, EntityNotFoundError +) + + +class ServiceError(Exception): + """Base exception for service errors""" + pass + + +class DatabaseSyncError(ServiceError): + """Raised when database sync fails""" + pass + + +class RelationError(ServiceError): + """Base exception for relation-specific errors""" + pass + + +class EntityService: + """Service for managing entities in the filesystem and database.""" + def __init__(self, project_path: Path, entity_repo: EntityRepository): + self.project_path = project_path + self.entity_repo = entity_repo + self.entities_path = project_path / "entities" + + async def _update_db_index(self, entity: Entity) -> DbEntity: + """Update database index with entity data.""" + entity_data = { + **entity.model_dump(), + "created_at": datetime.now(UTC), + "updated_at": datetime.now(UTC) + } + + # Observations will be handled by ObservationService + entity_data.pop('observations', None) # Remove observations if present + entity_data.pop('relations', None) # Remove relations if present + + # Try to find existing entity first + if await self.entity_repo.find_by_id(entity.id): + return await self.entity_repo.update(entity.id, entity_data) + else: + return await self.entity_repo.create(entity_data) + + async def create_entity(self, name: str, entity_type: str, + observations: Optional[list[str]] = None) -> Entity: + """Create a new entity.""" + # Convert string observations to Observation objects if provided + obs_list = [Observation(content=obs) for obs in (observations or [])] + + # Create entity (ID will be auto-generated) + entity = Entity( + name=name, + entity_type=entity_type, + observations=obs_list + ) + + # Step 1: Write to filesystem (source of truth) + await write_entity_file(self.entities_path, entity) + + # Step 2: Update database index + await self._update_db_index(entity) + + return entity + + async def get_entity(self, entity_id: str) -> Entity: + """Get entity by ID, reading from filesystem first.""" + # Read from filesystem (source of truth) + entity = await read_entity_file(self.entities_path, entity_id) + + # Update database index + await self._update_db_index(entity) + + return entity + + async def delete_entity(self, entity_id: str) -> bool: + """Delete entity from filesystem and database.""" + # Delete from filesystem first (source of truth) + await delete_entity_file(self.entities_path, entity_id) + + # Delete from database index + await self.entity_repo.delete(entity_id) + return True + + async def rebuild_index(self) -> None: + """Rebuild database index from filesystem contents.""" + if not self.entities_path.exists(): + return + + try: + entity_files = list(self.entities_path.glob("*.md")) + except Exception as e: + raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e + + for entity_file in entity_files: + try: + entity = await read_entity_file(self.entities_path, entity_file.stem) + await self._update_db_index(entity) + except Exception as e: + print(f"Warning: Failed to reindex {entity_file}: {str(e)}") + + +class ObservationService: + """Service for managing observations in the filesystem and database.""" + def __init__(self, project_path: Path, observation_repo: ObservationRepository): + self.project_path = project_path + self.entities_path = project_path / "entities" + self.observation_repo = observation_repo + + async def add_observation(self, entity: Entity, content: str, + context: Optional[str] = None) -> Observation: + """Add a new observation to an entity.""" + observation = Observation(content=content) + entity.observations.append(observation) + + # Update filesystem first (source of truth) + await write_entity_file(self.entities_path, entity) + + # Update database index + try: + db_observation = await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': content, + 'context': context, + 'created_at': datetime.now(UTC) + }) + return observation + except Exception as e: + raise DatabaseSyncError(f"Failed to sync observation to database: {str(e)}") from e + + async def search_observations(self, query: str) -> list[Observation]: + """ + Search for observations across all entities. + + Args: + query: Text to search for in observation content + + Returns: + List of matching observations with their entity contexts + """ + result = await self.observation_repo.execute_query( + select(DbObservation).filter( + DbObservation.content.contains(query) + ) + ) + return [ + Observation(content=obs.content) + for obs in result.scalars().all() + ] + + async def get_observations_by_context(self, context: str) -> list[Observation]: + """Get all observations with a specific context.""" + db_observations = await self.observation_repo.find_by_context(context) + return [ + Observation(content=obs.content) + for obs in db_observations + ] + + async def rebuild_observation_index(self) -> None: + """ + Rebuild the observation database index from filesystem contents. + Used for recovery or ensuring sync. + """ + # List all entity files + if not self.entities_path.exists(): + return + + try: + entity_files = list(self.entities_path.glob("*.md")) + except Exception as e: + raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e + + # Clear existing observation index + await self.observation_repo.execute_query(delete(DbObservation)) + + # Rebuild from each entity file + for entity_file in entity_files: + try: + entity = await read_entity_file(self.entities_path, entity_file.stem) + for obs in entity.observations: + await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': obs.content, + 'created_at': datetime.now(UTC) + }) + except Exception as e: + print(f"Warning: Failed to reindex observations for {entity_file}: {str(e)}") + + +class RelationService: + """ + Service for managing relations between entities. + Follows the "filesystem is source of truth" principle. + + Relations are stored in entity markdown files and indexed in the database + for efficient querying. + """ + + def __init__(self, project_path: Path, relation_repo: RelationRepository): + self.project_path = project_path + self.entities_path = project_path / "entities" + self.relation_repo = relation_repo + + async def create_relation(self, from_entity: Entity, to_entity: Entity, relation_type: str, + context: Optional[str] = None) -> Relation: + """ + Create a new relation between two entities. + + Args: + from_entity: Source entity + to_entity: Target entity + relation_type: Type of relation + context: Optional context for the relation + + Returns: + The created Relation + + Raises: + FileOperationError: If file operations fail + DatabaseSyncError: If database sync fails + """ + # Create new relation with actual Entity objects + relation = Relation( + from_entity=from_entity, + to_entity=to_entity, + relation_type=relation_type, + context=context + ) + + # Add relation to source entity's relations list + if not hasattr(from_entity, 'relations'): + from_entity.relations = [] + from_entity.relations.append(relation) + + # Update filesystem first (source of truth) + await write_entity_file(self.entities_path, from_entity) + + # Update database index + # model_dump will handle converting Entity refs to IDs + try: + db_data = relation.model_dump() + db_data['created_at'] = datetime.now(UTC) + await self.relation_repo.create(db_data) + return relation + except Exception as e: + raise DatabaseSyncError(f"Failed to sync relation to database: {str(e)}") from e + + async def get_entity_relations(self, entity: Entity) -> List[Relation]: + """ + Get all relations for an entity (both outgoing and incoming). + + Args: + entity: Entity to get relations for + + Returns: + List of relations where the entity is either source or target + """ + # Relations are stored in the entity object + return getattr(entity, 'relations', []) + + async def delete_relation(self, from_entity: Entity, relation_id: str) -> bool: + """ + Delete a relation from both filesystem and database. + + Args: + from_entity: Source entity containing the relation + relation_id: ID of the relation to delete + + Returns: + True if deletion was successful + + Raises: + RelationError: If relation cannot be found or deleted + """ + # Remove relation from entity's relations + if hasattr(from_entity, 'relations'): + from_entity.relations = [ + r for r in from_entity.relations + if r.id != relation_id + ] + + # Update filesystem first (source of truth) + await write_entity_file(self.entities_path, from_entity) + + # Remove from database index + await self.relation_repo.delete(relation_id) + return True + + async def rebuild_relation_index(self) -> None: + """ + Rebuild the relation database index from filesystem contents. + Used for recovery or ensuring sync. + """ + if not self.entities_path.exists(): + return + + try: + entity_files = list(self.entities_path.glob("*.md")) + except Exception as e: + raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e + + # Clear existing relation index + await self.relation_repo.execute_query(delete(DbRelation)) + + # Rebuild from each entity file + for entity_file in entity_files: + try: + entity = await read_entity_file(self.entities_path, entity_file.stem) + for relation in getattr(entity, 'relations', []): + db_data = relation.model_dump() + db_data['created_at'] = datetime.now(UTC) + await self.relation_repo.create(db_data) + except Exception as e: + print(f"Warning: Failed to reindex relations for {entity_file}: {str(e)}") diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index e28dd50d..a0383102 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,100 +1,63 @@ -"""Service for managing entities in both filesystem and database.""" +"""Service for managing entities in the database.""" from datetime import datetime, UTC from pathlib import Path from typing import Optional from basic_memory.models import Entity as DbEntity from basic_memory.repository import EntityRepository -from basic_memory.schemas import Entity, Observation -from basic_memory.fileio import ( - read_entity_file, write_entity_file, delete_entity_file, - FileOperationError -) +from basic_memory.schemas import Entity, EntityCreate from . import ServiceError, DatabaseSyncError - class EntityService: """ - Service for managing entities in the filesystem and database. - Follows the "filesystem is source of truth" principle. + Service for managing entities in the database. + File operations are handled by MemoryService. """ def __init__(self, project_path: Path, entity_repo: EntityRepository): self.project_path = project_path self.entity_repo = entity_repo - self.entities_path = project_path / "entities" - async def _update_db_index(self, entity: Entity) -> DbEntity: - """Update database index with entity data.""" - entity_data = { + async def create_entity(self, create_data: EntityCreate) -> Entity: + """Create a new entity in the database.""" + # Create Entity from EntityCreate data + entity = Entity.from_create(create_data) + + # Create DB record + db_data = { **entity.model_dump(), "created_at": datetime.now(UTC), "updated_at": datetime.now(UTC) } + await self.entity_repo.create(db_data) - # Remove fields handled by other services - entity_data.pop('observations', None) - entity_data.pop('relations', None) - - # Try to find existing entity first - if await self.entity_repo.find_by_id(entity.id): - return await self.entity_repo.update(entity.id, entity_data) - else: - return await self.entity_repo.create(entity_data) - - async def create_entity(self, name: str, entity_type: str, - observations: Optional[list[str]] = None) -> Entity: - """Create a new entity.""" - # Convert string observations to Observation objects if provided - obs_list = [Observation(content=obs) for obs in (observations or [])] - - # Create entity (ID will be auto-generated) - entity = Entity( - name=name, - entity_type=entity_type, - observations=obs_list - ) - - # Step 1: Write to filesystem (source of truth) - await write_entity_file(self.entities_path, entity) - - # Step 2: Update database index - await self._update_db_index(entity) - return entity async def get_entity(self, entity_id: str) -> Entity: - """Get entity by ID, reading from filesystem first.""" - # Read from filesystem (source of truth) - entity = await read_entity_file(self.entities_path, entity_id) - - # Update database index - await self._update_db_index(entity) + """Get entity by ID.""" + db_entity = await self.entity_repo.find_by_id(entity_id) + if not db_entity: + raise ServiceError(f"Entity not found: {entity_id}") - return entity + return Entity( + id=db_entity.id, + name=db_entity.name, + entity_type=db_entity.entity_type + ) + + async def get_by_name(self, name: str) -> Entity: + """Get entity by name.""" + db_entity = await self.entity_repo.find_by_name(name) + if not db_entity: + raise ServiceError(f"Entity not found: {name}") + + return Entity( + id=db_entity.id, + name=db_entity.name, + entity_type=db_entity.entity_type + ) async def delete_entity(self, entity_id: str) -> bool: - """Delete entity from filesystem and database.""" - # Delete from filesystem first (source of truth) - await delete_entity_file(self.entities_path, entity_id) - - # Delete from database index + """Delete entity from database.""" await self.entity_repo.delete(entity_id) - return True - - async def rebuild_index(self) -> None: - """Rebuild database index from filesystem contents.""" - if not self.entities_path.exists(): - return - - try: - entity_files = list(self.entities_path.glob("*.md")) - except Exception as e: - raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e - - for entity_file in entity_files: - try: - entity = await read_entity_file(self.entities_path, entity_file.stem) - await self._update_db_index(entity) - except Exception as e: - print(f"Warning: Failed to reindex {entity_file}: {str(e)}") + return True \ No newline at end of file diff --git a/src/basic_memory/services/memory_service.py b/src/basic_memory/services/memory_service.py index ebf7d19b..26439b43 100644 --- a/src/basic_memory/services/memory_service.py +++ b/src/basic_memory/services/memory_service.py @@ -1,120 +1,50 @@ +"""Service for orchestrating entity, relation, and observation operations.""" from typing import List, Dict, Any, Optional from pathlib import Path -from ..schemas import Entity, Relation, Observation +from ..schemas import Entity, EntityCreate, Relation, RelationCreate, Observation +from ..fileio import write_entity_file, read_entity_file, delete_entity_file from .entity_service import EntityService from .relation_service import RelationService -from .observation_service import ObservationService class MemoryService: - """Orchestrates entity, relation, and observation operations.""" + """Orchestrates entity, relation, and observation operations with filesystem handling.""" def __init__(self, project_path: Optional[Path] = None): self.project_path = project_path + self.entities_path = project_path / "entities" if project_path else None + # Initialize with repos when we add them self.entity_service = EntityService() self.relation_service = RelationService() - self.observation_service = ObservationService() async def create_entities(self, entities_data: List[Dict[str, Any]]) -> List[Entity]: """Create multiple entities with their observations.""" - results = [] - for data in entities_data: - # Convert to Pydantic model for validation - entity = Entity( - name=data["name"], - entity_type=data["entityType"] - ) + async def create_and_write(data: Dict[str, Any]) -> Entity: + create_data = EntityCreate.model_validate(data) + entity = await self.entity_service.create_entity(create_data) + await write_entity_file(self.entities_path, entity) + return entity - # Create the entity - created = await self.entity_service.create_entity(entity) - - # Add observations if any - if observations := data.get("observations"): - obs_models = [Observation(content=obs) for obs in observations] - await self.observation_service.add_observations(created.id, obs_models) - created.observations.extend(obs_models) - - results.append(created) - - return results + return [await create_and_write(data) for data in entities_data] async def create_relations(self, relations_data: List[Dict[str, Any]]) -> List[Relation]: """Create multiple relations between entities.""" - results = [] - for data in relations_data: - # Get the entities - from_entity = await self.entity_service.get_entity(data["from"]) - to_entity = await self.entity_service.get_by_name(data["to"]) + async def create_and_write_relation(data: Dict[str, Any]) -> Relation: + create_data = RelationCreate.model_validate(data) + + # Get the actual entities + from_entity = await self.entity_service.get_by_name(create_data.from_) + to_entity = await self.entity_service.get_by_name(create_data.to) # Create relation - relation = Relation( + relation = await self.relation_service.create_relation( + create_data=create_data, from_entity=from_entity, - to_entity=to_entity, - relation_type=data["relationType"] + to_entity=to_entity ) - created = await self.relation_service.create(relation) - results.append(created) - return results - - async def add_observations(self, observations_data: List[Dict[str, Any]]) -> None: - """Add observations to existing entities.""" - for data in observations_data: - entity = await self.entity_service.get_by_name(data["entityName"]) - observations = [Observation(content=content) for content in data["contents"]] - await self.observation_service.add_observations(entity.id, observations) - - async def delete_entities(self, entity_names: List[str]) -> None: - """Delete multiple entities and their associated data.""" - for name in entity_names: - entity = await self.entity_service.get_by_name(name) - await self.entity_service.delete(entity.id) - - async def delete_observations(self, deletions: List[Dict[str, Any]]) -> None: - """Delete specific observations from entities.""" - for deletion in deletions: - entity = await self.entity_service.get_by_name(deletion["entityName"]) - observations = [Observation(content=content) for content in deletion["observations"]] - await self.observation_service.delete_observations(entity.id, observations) - - async def delete_relations(self, relations: List[Dict[str, Any]]) -> None: - """Delete specific relations between entities.""" - for data in relations: - # Get the entities - from_entity = await self.entity_service.get_by_name(data["from"]) - to_entity = await self.entity_service.get_by_name(data["to"]) + # Write updated from_entity to file + await write_entity_file(self.entities_path, from_entity) + return relation - await self.relation_service.delete_relation( - from_entity.id, - to_entity.id, - data["relationType"] - ) - - async def read_graph(self) -> Dict[str, Any]: - """Read the entire knowledge graph.""" - entities = await self.entity_service.get_all() - return { - "entities": [entity.model_dump() for entity in entities] - # Relations are included in entity.model_dump() - } - - async def search_nodes(self, query: str) -> Dict[str, Any]: - """Search for nodes in the knowledge graph.""" - results = await self.entity_service.search(query) - return { - "matches": [entity.model_dump() for entity in results], - "query": query - } - - async def open_nodes(self, names: List[str]) -> Dict[str, Any]: - """Get specific nodes and their relationships.""" - entities = [] - for name in names: - entity = await self.entity_service.get_by_name(name) - if entity: - entities.append(entity) - - return { - "entities": [entity.model_dump() for entity in entities] - # Relations between these entities are included in model_dump() - } \ No newline at end of file + return [await create_and_write_relation(data) for data in relations_data] \ No newline at end of file diff --git a/src/basic_memory/services/observation_service.py b/src/basic_memory/services/observation_service.py index 88121590..8c967691 100644 --- a/src/basic_memory/services/observation_service.py +++ b/src/basic_memory/services/observation_service.py @@ -8,63 +8,39 @@ from sqlalchemy import select, delete from basic_memory.models import Observation as DbObservation from basic_memory.repository import ObservationRepository from basic_memory.schemas import Entity, Observation -from basic_memory.fileio import ( - write_entity_file, read_entity_file, - FileOperationError -) from . import ServiceError, DatabaseSyncError class ObservationService: """ - Service for managing observations in the filesystem and database. - Follows the "filesystem is source of truth" principle. - - Observations are stored in entity markdown files and indexed in the database - for efficient querying. + Service for managing observations in the database. + File operations are handled by MemoryService. """ def __init__(self, project_path: Path, observation_repo: ObservationRepository): self.project_path = project_path - self.entities_path = project_path / "entities" self.observation_repo = observation_repo - async def add_observation(self, entity: Entity, content: str, - context: Optional[str] = None) -> Observation: + async def add_observations(self, entity: Entity, observations: List[Observation]) -> List[Observation]: """ - Add a new observation to an entity. - - Args: - entity: Entity to add observation to - content: Content of the observation - context: Optional context for the observation - - Returns: - The created Observation - - Raises: - FileOperationError: If file operations fail - DatabaseSyncError: If database sync fails + Add multiple observations to an entity. + Updates database indexes only - filesystem write handled by MemoryService. """ - # Create new observation - observation = Observation(content=content) - entity.observations.append(observation) - - # Update filesystem first (source of truth) - await write_entity_file(self.entities_path, entity) - # Update database index - try: - db_observation = await self.observation_repo.create({ - 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", - 'entity_id': entity.id, - 'content': content, - 'context': context, - 'created_at': datetime.now(UTC) - }) - return observation - except Exception as e: - raise DatabaseSyncError(f"Failed to sync observation to database: {str(e)}") from e + for observation in observations: + try: + await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': observation.content, + 'created_at': datetime.now(UTC) + }) + except Exception as e: + raise DatabaseSyncError(f"Failed to sync observation to database: {str(e)}") from e + + # Add to entity in memory + entity.observations.extend(observations) + return observations async def search_observations(self, query: str) -> List[Observation]: """ @@ -94,33 +70,21 @@ class ObservationService: for obs in db_observations ] - async def rebuild_observation_index(self) -> None: + async def rebuild_observation_index(self, entity: Entity) -> None: """ - Rebuild the observation database index from filesystem contents. + Rebuild the observation database index for a specific entity. Used for recovery or ensuring sync. """ - # List all entity files - if not self.entities_path.exists(): - return - - try: - entity_files = list(self.entities_path.glob("*.md")) - except Exception as e: - raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e - - # Clear existing observation index - await self.observation_repo.execute_query(delete(DbObservation)) + # Clear existing observations for this entity + await self.observation_repo.execute_query( + delete(DbObservation).where(DbObservation.entity_id == entity.id) + ) - # Rebuild from each entity file - for entity_file in entity_files: - try: - entity = await read_entity_file(self.entities_path, entity_file.stem) - for obs in entity.observations: - await self.observation_repo.create({ - 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", - 'entity_id': entity.id, - 'content': obs.content, - 'created_at': datetime.now(UTC) - }) - except Exception as e: - print(f"Warning: Failed to reindex observations for {entity_file}: {str(e)}") + # Rebuild from entity's observations + for obs in entity.observations: + await self.observation_repo.create({ + 'id': f"{entity.id}-obs-{uuid4().hex[:8]}", + 'entity_id': entity.id, + 'content': obs.content, + 'created_at': datetime.now(UTC) + }) \ No newline at end of file diff --git a/src/basic_memory/services/relation_service.py b/src/basic_memory/services/relation_service.py index b9757988..289d53c5 100644 --- a/src/basic_memory/services/relation_service.py +++ b/src/basic_memory/services/relation_service.py @@ -1,69 +1,35 @@ -"""Service for managing relations between entities.""" +"""Service for managing relations in the database.""" from datetime import datetime, UTC from pathlib import Path -from typing import Optional, List -from sqlalchemy import delete +from typing import Optional from basic_memory.models import Relation as DbRelation from basic_memory.repository import RelationRepository -from basic_memory.schemas import Entity, Relation -from basic_memory.fileio import ( - write_entity_file, read_entity_file, - FileOperationError -) +from basic_memory.schemas import Entity, Relation, RelationCreate from . import ServiceError, DatabaseSyncError, RelationError class RelationService: """ - Service for managing relations between entities. - Follows the "filesystem is source of truth" principle. - - Relations are stored in entity markdown files and indexed in the database - for efficient querying. + Service for managing relations in the database. + File operations are handled by MemoryService. """ def __init__(self, project_path: Path, relation_repo: RelationRepository): self.project_path = project_path - self.entities_path = project_path / "entities" self.relation_repo = relation_repo - - async def create_relation(self, from_entity: Entity, to_entity: Entity, relation_type: str, - context: Optional[str] = None) -> Relation: - """ - Create a new relation between two entities. - - Args: - from_entity: Source entity - to_entity: Target entity - relation_type: Type of relation - context: Optional context for the relation - - Returns: - The created Relation - - Raises: - FileOperationError: If file operations fail - DatabaseSyncError: If database sync fails - """ + + async def create_relation(self, create_data: RelationCreate, from_entity: Entity, to_entity: Entity) -> Relation: + """Create a new relation between two entities.""" # Create new relation with actual Entity objects - relation = Relation( - from_entity=from_entity, - to_entity=to_entity, - relation_type=relation_type, - context=context - ) + relation = Relation.from_create(create_data, from_entity, to_entity) # Add relation to source entity's relations list if not hasattr(from_entity, 'relations'): from_entity.relations = [] from_entity.relations.append(relation) - # Update filesystem first (source of truth) - await write_entity_file(self.entities_path, from_entity) - # Update database index - # model_dump will handle converting Entity refs to IDs try: db_data = relation.model_dump() db_data['created_at'] = datetime.now(UTC) @@ -72,69 +38,21 @@ class RelationService: except Exception as e: raise DatabaseSyncError(f"Failed to sync relation to database: {str(e)}") from e - async def get_entity_relations(self, entity: Entity) -> List[Relation]: - """ - Get all relations for an entity (outgoing relations). - - Args: - entity: Entity to get relations for - - Returns: - List of relations where the entity is the source - """ - return getattr(entity, 'relations', []) - - async def delete_relation(self, from_entity: Entity, relation_id: str) -> bool: - """ - Delete a relation from both filesystem and database. - - Args: - from_entity: Source entity containing the relation - relation_id: ID of the relation to delete - - Returns: - True if deletion was successful - - Raises: - RelationError: If relation cannot be found or deleted - """ - # Remove relation from entity's relations + async def delete_relation(self, from_entity: Entity, to_entity: Entity, relation_type: str) -> bool: + """Delete a specific relation between entities.""" + # Find and remove the relation from the entity's relations if hasattr(from_entity, 'relations'): from_entity.relations = [ r for r in from_entity.relations - if r.id != relation_id + if not (r.to_entity.id == to_entity.id and r.relation_type == relation_type) ] - - # Update filesystem first (source of truth) - await write_entity_file(self.entities_path, from_entity) # Remove from database index - await self.relation_repo.delete(relation_id) - return True - - async def rebuild_relation_index(self) -> None: - """ - Rebuild the relation database index from filesystem contents. - Used for recovery or ensuring sync. - """ - if not self.entities_path.exists(): - return - - try: - entity_files = list(self.entities_path.glob("*.md")) - except Exception as e: - raise FileOperationError(f"Failed to read entities directory: {str(e)}") from e - - # Clear existing relation index - await self.relation_repo.execute_query(delete(DbRelation)) - - # Rebuild from each entity file - for entity_file in entity_files: - try: - entity = await read_entity_file(self.entities_path, entity_file.stem) - for relation in getattr(entity, 'relations', []): - db_data = relation.model_dump() - db_data['created_at'] = datetime.now(UTC) - await self.relation_repo.create(db_data) - except Exception as e: - print(f"Warning: Failed to reindex relations for {entity_file}: {str(e)}") + await self.relation_repo.execute_query( + delete(DbRelation).where( + (DbRelation.from_id == from_entity.id) & + (DbRelation.to_id == to_entity.id) & + (DbRelation.relation_type == relation_type) + ) + ) + return True \ No newline at end of file