From de1cbf29c3c0d042c04d81df71f72078e089d3d5 Mon Sep 17 00:00:00 2001 From: phernandez Date: Fri, 3 Jan 2025 16:08:07 -0600 Subject: [PATCH] fix imports for service.sync --- src/basic_memory/cli/commands/status.py | 2 +- src/basic_memory/cli/commands/sync.py | 88 +++++++++++++++++++ src/basic_memory/cli/main.py | 4 +- src/basic_memory/services/__init__.py | 2 - src/basic_memory/services/sync/__init__.py | 6 ++ .../services/sync/sync_service.py | 21 +++-- tests/conftest.py | 2 +- .../services/sync/test_file_change_scanner.py | 2 +- tests/services/sync/test_sync_service.py | 63 +++++-------- 9 files changed, 135 insertions(+), 55 deletions(-) create mode 100644 src/basic_memory/cli/commands/sync.py diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index c8595acc..c5a01bda 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -14,7 +14,7 @@ from basic_memory.cli.app import app from basic_memory.config import config from basic_memory.db import DatabaseType from basic_memory.repository import DocumentRepository, EntityRepository -from basic_memory.services import FileChangeScanner +from basic_memory.services.sync import FileChangeScanner from basic_memory.services.sync.utils import SyncReport # Create rich console diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py new file mode 100644 index 00000000..a888c2a8 --- /dev/null +++ b/src/basic_memory/cli/commands/sync.py @@ -0,0 +1,88 @@ +"""Command module for basic-memory sync operations.""" +from pathlib import Path +from typing import Optional + +import typer +import asyncio +from loguru import logger + +from basic_memory.cli.app import app +from basic_memory import db +from basic_memory.config import config +from basic_memory.db import DatabaseType +from basic_memory.repository import DocumentRepository, EntityRepository +from basic_memory.services import ( + DocumentService, + EntityService, +) +from basic_memory.markdown import KnowledgeParser +from basic_memory.services.sync import SyncService, FileChangeScanner, KnowledgeSyncService + + +async def get_sync_service(db_type=DatabaseType.FILESYSTEM): + """Get sync service instance with all dependencies.""" + async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as ( + engine, + session_maker, + ): + # Initialize repositories + document_repository = DocumentRepository(session_maker) + entity_repository = EntityRepository(session_maker) + + # Initialize scanner + file_change_scanner = FileChangeScanner(document_repository, entity_repository) + + # Initialize services + document_service = DocumentService(document_repository) + entity_service = EntityService(entity_repository) + knowledge_sync_service = KnowledgeSyncService(entity_service) + knowledge_parser = KnowledgeParser() + + # Create sync service + sync_service = SyncService( + scanner=file_change_scanner, + document_service=document_service, + knowledge_sync_service=knowledge_sync_service, + knowledge_parser=knowledge_parser, + ) + + return sync_service + + +async def run_sync(root_dir: Path): + """Run sync operation.""" + sync_service = await get_sync_service() + await sync_service.sync(root_dir) + + +@app.command() +def sync( + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Show detailed sync information.", + ), +) -> None: + """Sync knowledge files with the database. + + This command syncs both documents and knowledge files with the database, + using a two-pass strategy for knowledge files to handle relations correctly. + + Use 'basic-memory status' to preview changes before syncing. + """ + try: + # Get project directory + project_dir = config.home + logger.info(f"Syncing directory: {project_dir}") + + # Run sync + asyncio.run(run_sync(project_dir)) + + if verbose: + logger.info("Sync completed successfully") + + except Exception as e: + logger.exception(f"Sync failed: {e}") + typer.echo(f"Error during sync: {e}", err=True) + raise typer.Exit(1) diff --git a/src/basic_memory/cli/main.py b/src/basic_memory/cli/main.py index 9dfc1f7c..b064bfa4 100644 --- a/src/basic_memory/cli/main.py +++ b/src/basic_memory/cli/main.py @@ -8,8 +8,8 @@ from basic_memory.cli.app import app from basic_memory.cli.commands.init import init # Register commands -from basic_memory.cli.commands import init, status -__all__ = ["init", "status"] +from basic_memory.cli.commands import init, status, sync +__all__ = ["init", "status", "sync"] from basic_memory.config import config diff --git a/src/basic_memory/services/__init__.py b/src/basic_memory/services/__init__.py index 3578eb7b..ddd02b32 100644 --- a/src/basic_memory/services/__init__.py +++ b/src/basic_memory/services/__init__.py @@ -1,7 +1,6 @@ """Services package.""" from .document_service import DocumentService -from .sync.file_change_scanner import FileChangeScanner from .entity_service import EntityService from .file_service import FileService from .knowledge import KnowledgeService @@ -16,6 +15,5 @@ __all__ = [ "FileService", "ObservationService", "RelationService", - "FileChangeScanner", "KnowledgeService", ] diff --git a/src/basic_memory/services/sync/__init__.py b/src/basic_memory/services/sync/__init__.py index e69de29b..90c03896 100644 --- a/src/basic_memory/services/sync/__init__.py +++ b/src/basic_memory/services/sync/__init__.py @@ -0,0 +1,6 @@ +from .file_change_scanner import FileChangeScanner +from .knowledge_sync_service import KnowledgeSyncService +from .sync_service import SyncService + +__all__ = ["SyncService", "FileChangeScanner", "KnowledgeSyncService"] + diff --git a/src/basic_memory/services/sync/sync_service.py b/src/basic_memory/services/sync/sync_service.py index 87908921..b786686c 100644 --- a/src/basic_memory/services/sync/sync_service.py +++ b/src/basic_memory/services/sync/sync_service.py @@ -3,14 +3,15 @@ from pathlib import Path from loguru import logger -from basic_memory.services import FileChangeScanner, EntityService, DocumentService +from basic_memory.services import DocumentService +from basic_memory.services.sync import FileChangeScanner from basic_memory.markdown import KnowledgeParser from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService class SyncService: """Syncs documents and knowledge files with database. - + Implements two-pass sync strategy for knowledge files to handle relations: 1. First pass creates/updates entities without relations 2. Second pass processes relations after all entities exist @@ -46,7 +47,9 @@ class SyncService: await self.document_service.create_document(path_id=path, content=content) else: logger.debug(f"Updating document: {path}") - await self.document_service.update_document_by_path_id(path_id=path, content=content) + await self.document_service.update_document_by_path_id( + path_id=path, content=content + ) async def sync_knowledge(self, directory: Path) -> None: """Sync knowledge files with database.""" @@ -73,12 +76,16 @@ class SyncService: else: path_id = entity_markdown.frontmatter.id logger.debug(f"Updating entity_markdown: {path_id}") - await self.knowledge_sync_service.update_entity_and_observations(path_id, entity_markdown) + await self.knowledge_sync_service.update_entity_and_observations( + path_id, entity_markdown + ) # Second pass: Process relations for file_path, entity_markdown in parsed_entities.items(): logger.debug(f"Updating relations for: {file_path}") - await self.knowledge_sync_service.update_entity_relations(entity_markdown, checksum=changes.checksums[file_path]) + await self.knowledge_sync_service.update_entity_relations( + entity_markdown, checksum=changes.checksums[file_path] + ) async def sync(self, root_dir: Path) -> None: """Sync all files with database.""" @@ -86,8 +93,8 @@ class SyncService: docs_dir = root_dir / "documents" if docs_dir.exists(): await self.sync_documents(docs_dir) - + # Then sync knowledge files knowledge_dir = root_dir / "knowledge" if knowledge_dir.exists(): - await self.sync_knowledge(knowledge_dir) \ No newline at end of file + await self.sync_knowledge(knowledge_dir) diff --git a/tests/conftest.py b/tests/conftest.py index 858f76bf..12e8d6b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,8 +23,8 @@ from basic_memory.services import ( ObservationService, RelationService, DocumentService, - FileChangeScanner, ) +from basic_memory.services.sync import FileChangeScanner from basic_memory.services.activity_service import ActivityService from basic_memory.services.file_service import FileService from basic_memory.services import KnowledgeService diff --git a/tests/services/sync/test_file_change_scanner.py b/tests/services/sync/test_file_change_scanner.py index 0471cf38..cf188fcb 100644 --- a/tests/services/sync/test_file_change_scanner.py +++ b/tests/services/sync/test_file_change_scanner.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import AsyncGenerator from basic_memory.repository import DocumentRepository, EntityRepository -from basic_memory.services import FileChangeScanner +from basic_memory.services.sync import FileChangeScanner from basic_memory.services.sync.utils import DbState from basic_memory.utils.file_utils import compute_checksum from basic_memory.models import Document, Entity diff --git a/tests/services/sync/test_sync_service.py b/tests/services/sync/test_sync_service.py index 23165983..f7c85a85 100644 --- a/tests/services/sync/test_sync_service.py +++ b/tests/services/sync/test_sync_service.py @@ -3,15 +3,11 @@ import asyncio from pathlib import Path import pytest -import pytest_asyncio -from loguru import logger from basic_memory.config import ProjectConfig -from basic_memory.services import DocumentService, EntityService, FileChangeScanner -from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService +from basic_memory.services import EntityService from basic_memory.services.sync.sync_service import SyncService -from basic_memory.markdown import KnowledgeParser -from basic_memory.models import Document, Entity, Observation +from basic_memory.models import Entity async def create_test_file(path: Path, content: str = "test content") -> None: @@ -21,13 +17,10 @@ async def create_test_file(path: Path, content: str = "test content") -> None: @pytest.mark.asyncio -async def test_sync_empty_directories( - sync_service: SyncService, - test_config: ProjectConfig -): +async def test_sync_empty_directories(sync_service: SyncService, test_config: ProjectConfig): """Test syncing empty directories.""" await sync_service.sync(test_config.home) - + # Should not raise exceptions for empty dirs assert (test_config.documents_dir).exists() assert (test_config.knowledge_dir).exists() @@ -35,8 +28,7 @@ async def test_sync_empty_directories( @pytest.mark.asyncio async def test_sync_file_modified_during_sync( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test handling of files that change during sync process.""" # Create initial files @@ -49,10 +41,7 @@ async def test_sync_file_modified_during_sync( doc_path.write_text("Modified during sync") # Run sync and modification concurrently - await asyncio.gather( - sync_service.sync(test_config.home), - modify_file() - ) + await asyncio.gather(sync_service.sync(test_config.home), modify_file()) # Verify final state doc = await sync_service.document_service.repository.find_by_path_id("changing.md") @@ -63,9 +52,7 @@ async def test_sync_file_modified_during_sync( @pytest.mark.asyncio async def test_sync_null_checksum_cleanup( - sync_service: SyncService, - test_config: ProjectConfig, - entity_service: EntityService + sync_service: SyncService, test_config: ProjectConfig, entity_service: EntityService ): """Test handling of entities with null checksums from incomplete syncs.""" # Create entity with null checksum (simulating incomplete sync) @@ -74,7 +61,7 @@ async def test_sync_null_checksum_cleanup( name="Incomplete", entity_type="concept", file_path="concept/incomplete.md", - checksum=None # Null checksum + checksum=None, # Null checksum ) await entity_service.repository.add(entity) @@ -102,10 +89,7 @@ modified: 2024-01-01 @pytest.mark.asyncio -async def test_sync_mixed_document_types( - sync_service: SyncService, - test_config: ProjectConfig -): +async def test_sync_mixed_document_types(sync_service: SyncService, test_config: ProjectConfig): """Test handling documents and knowledge files with similar paths.""" # Create a document doc_content = "# Regular Document" @@ -139,17 +123,11 @@ modified: 2024-01-01 @pytest.mark.asyncio -async def test_sync_performance_large_files( - sync_service: SyncService, - test_config: ProjectConfig -): +async def test_sync_performance_large_files(sync_service: SyncService, test_config: ProjectConfig): """Test sync performance with larger files.""" # Create a large document with many lines large_doc = ["Line " + str(i) for i in range(1000)] - await create_test_file( - test_config.documents_dir / "large.md", - "\n".join(large_doc) - ) + await create_test_file(test_config.documents_dir / "large.md", "\n".join(large_doc)) # Create a knowledge file with many observations observations = [f"- Observation {i}" for i in range(100)] @@ -176,13 +154,16 @@ modified: 2024-01-01 doc = await sync_service.document_service.repository.find_by_path_id("large.md") assert doc is not None - entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/large") + entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/large" + ) assert entity is not None assert len(entity.observations) == 100 # Basic performance check - should sync in reasonable time assert duration < 5 # Should complete in under 5 seconds + # skip for now - until we handle concurrency with db # @pytest.mark.asyncio @@ -193,34 +174,34 @@ modified: 2024-01-01 # """Test concurrent syncs maintain database consistency.""" # doc1_path = test_config.documents_dir / "doc1.md" # doc2_path = test_config.documents_dir / "doc2.md" -# +# # await create_test_file(doc1_path, "Doc 1 content") # await create_test_file(doc2_path, "Doc 2 content") -# +# # # Run concurrent syncs # results = await asyncio.gather( # sync_service.sync(test_config.home), # sync_service.sync(test_config.home), # return_exceptions=True # ) -# +# # # Check no exceptions were raised # for r in results: # if isinstance(r, Exception): # print(r) # assert not isinstance(r, Exception) -# +# # # Verify database consistency # docs = await sync_service.document_service.repository.find_all() # assert len(docs) == 2 # No duplicates # assert {d.path_id for d in docs} == {"doc1.md", "doc2.md"} -# +# # # Both files should have valid checksums # for doc in docs: # assert doc.checksum is not None -# +# # # Running another sync should not change anything # await sync_service.sync(test_config.home) # docs_after = await sync_service.document_service.repository.find_all() # assert len(docs_after) == 2 -# assert {d.path_id for d in docs_after} == {"doc1.md", "doc2.md"} \ No newline at end of file +# assert {d.path_id for d in docs_after} == {"doc1.md", "doc2.md"}