refactor sync

This commit is contained in:
phernandez
2025-01-11 00:49:11 -06:00
parent 48aa3faa02
commit c2ea73342f
6 changed files with 29 additions and 34 deletions
+4 -4
View File
@@ -29,7 +29,7 @@ from basic_memory.services import (
EntityService,
)
from basic_memory.services.search_service import SearchService
from basic_memory.sync import SyncService, FileChangeScanner, KnowledgeSyncService
from basic_memory.sync import SyncService, FileChangeScanner, EntitySyncService
from basic_memory.sync.utils import SyncReport
from basic_memory.utils.file_utils import ParseError
@@ -60,7 +60,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
# Initialize services
entity_service = EntityService(entity_repository)
knowledge_sync_service = KnowledgeSyncService(
knowledge_sync_service = EntitySyncService(
entity_repository, observation_repository, relation_repository
)
entity_parser = EntityParser(config.home)
@@ -70,7 +70,7 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
# Create sync service
sync_service = SyncService(
scanner=file_change_scanner,
knowledge_sync_service=knowledge_sync_service,
entity_sync_service=knowledge_sync_service,
entity_parser=entity_parser,
search_service=search_service,
)
@@ -212,7 +212,7 @@ async def run_sync(verbose: bool = False):
raise typer.Exit(1)
# Sync
knowledge_changes = await sync_service.sync(config)
knowledge_changes = await sync_service.sync(config.knowledge_dir)
# Display results
if verbose:
+2 -2
View File
@@ -1,6 +1,6 @@
from .file_change_scanner import FileChangeScanner
from .knowledge_sync_service import KnowledgeSyncService
from .entity_sync_service import EntitySyncService
from .sync_service import SyncService
__all__ = ["SyncService", "FileChangeScanner", "KnowledgeSyncService"]
__all__ = ["SyncService", "FileChangeScanner", "EntitySyncService"]
+4 -2
View File
@@ -52,7 +52,7 @@ class EntitySyncService:
async def delete_entity_by_file_path(self, file_path: str) -> bool:
return await self.entity_repository.delete_by_file_path(file_path)
async def create_entity_and_observations(
async def create_entity_from_markdown(
self, file_path: str, markdown: EntityMarkdown
) -> EntityModel:
"""First pass: Create entity and observations only.
@@ -62,7 +62,9 @@ class EntitySyncService:
"""
logger.debug(f"Creating entity without relations: {markdown.frontmatter.id}")
model = entity_model_from_markdown(file_path, markdown)
model.checksum = None # Mark as incomplete sync
# Mark as incomplete sync
model.checksum = None
return await self.entity_repository.add(model)
async def update_entity_and_observations(
+5 -12
View File
@@ -4,11 +4,10 @@ from pathlib import Path
from loguru import logger
from basic_memory.config import ProjectConfig
from basic_memory.markdown import EntityParser
from basic_memory.services.search_service import SearchService
from basic_memory.sync import FileChangeScanner
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
from basic_memory.sync.entity_sync_service import EntitySyncService
from basic_memory.sync.utils import SyncReport
@@ -23,17 +22,16 @@ class SyncService:
def __init__(
self,
scanner: FileChangeScanner,
knowledge_sync_service: KnowledgeSyncService,
entity_sync_service: EntitySyncService,
entity_parser: EntityParser,
search_service: SearchService,
):
self.scanner = scanner
self.knowledge_sync_service = knowledge_sync_service
self.knowledge_sync_service = entity_sync_service
self.knowledge_parser = entity_parser
self.search_service = search_service
async def sync_knowledge(self, directory: Path) -> SyncReport:
async def sync(self, directory: Path) -> SyncReport:
"""Sync knowledge files with database."""
changes = await self.scanner.find_knowledge_changes(directory)
logger.info(f"Found {changes.total_changes} knowledge changes")
@@ -54,7 +52,7 @@ class SyncService:
for file_path, entity_markdown in parsed_entities.items():
if file_path in changes.new:
logger.debug(f"Creating new entity_markdown: {file_path}")
await self.knowledge_sync_service.create_entity_and_observations(
await self.knowledge_sync_service.create_entity_from_markdown(
file_path, entity_markdown
)
else:
@@ -74,8 +72,3 @@ class SyncService:
await self.search_service.index_entity(entity)
return changes
async def sync(self, config: ProjectConfig) -> SyncReport:
"""Sync all files with database."""
knowledge_changes = await self.sync_knowledge(config.knowledge_dir)
return knowledge_changes
+5 -5
View File
@@ -28,7 +28,7 @@ from basic_memory.services.activity_service import ActivityService
from basic_memory.services.file_service import FileService
from basic_memory.services.search_service import SearchService
from basic_memory.sync import FileChangeScanner
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
from basic_memory.sync.entity_sync_service import EntitySyncService
from basic_memory.sync.sync_service import SyncService
@@ -162,14 +162,14 @@ async def knowledge_sync_service(
entity_repository: EntityRepository,
observation_repository: ObservationRepository,
relation_repository: RelationRepository,
) -> KnowledgeSyncService:
) -> EntitySyncService:
"""Create EntitySyncService with repository."""
return KnowledgeSyncService(entity_repository, observation_repository, relation_repository)
return EntitySyncService(entity_repository, observation_repository, relation_repository)
@pytest_asyncio.fixture
async def sync_service(
knowledge_sync_service: KnowledgeSyncService,
knowledge_sync_service: EntitySyncService,
file_change_scanner: FileChangeScanner,
entity_parser: EntityParser,
search_service: SearchService,
@@ -177,7 +177,7 @@ async def sync_service(
"""Create sync service for testing."""
return SyncService(
scanner=file_change_scanner,
knowledge_sync_service=knowledge_sync_service,
entity_sync_service=knowledge_sync_service,
entity_parser=entity_parser,
search_service=search_service,
)
+9 -9
View File
@@ -13,7 +13,7 @@ from basic_memory.markdown.schemas import (
Relation as MarkdownRelation,
)
from basic_memory.models import Entity as EntityModel
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
from basic_memory.sync.entity_sync_service import EntitySyncService
@pytest_asyncio.fixture
@@ -55,11 +55,11 @@ def test_markdown(test_frontmatter, test_content) -> EntityMarkdown:
@pytest.mark.asyncio
async def test_create_entity_without_relations(
knowledge_sync_service: KnowledgeSyncService, test_markdown: EntityMarkdown
knowledge_sync_service: EntitySyncService, test_markdown: EntityMarkdown
):
"""Test first pass creation without relations."""
# Create entity first pass
entity = await knowledge_sync_service.create_entity_and_observations("test.md", test_markdown)
entity = await knowledge_sync_service.create_entity_from_markdown("test.md", test_markdown)
# Check basic fields
assert entity.name == "Test Entity"
@@ -81,11 +81,11 @@ async def test_create_entity_without_relations(
@pytest.mark.asyncio
async def test_update_entity_without_relations(
knowledge_sync_service: KnowledgeSyncService, test_markdown: EntityMarkdown
knowledge_sync_service: EntitySyncService, test_markdown: EntityMarkdown
):
"""Test first pass update."""
# First create entity
entity = await knowledge_sync_service.create_entity_and_observations("test.md", test_markdown)
entity = await knowledge_sync_service.create_entity_from_markdown("test.md", test_markdown)
# Modify markdown content
test_markdown.frontmatter.title = "Updated Title"
@@ -109,11 +109,11 @@ async def test_update_entity_without_relations(
@pytest.mark.asyncio
async def test_update_entity_relations(
knowledge_sync_service: KnowledgeSyncService, test_markdown: EntityMarkdown
knowledge_sync_service: EntitySyncService, test_markdown: EntityMarkdown
):
"""Test second pass relation updates."""
# Create main entity first
entity = await knowledge_sync_service.create_entity_and_observations("test.md", test_markdown)
entity = await knowledge_sync_service.create_entity_from_markdown("test.md", test_markdown)
# Create target entities that relations point to
other_entity = EntityModel(
@@ -157,7 +157,7 @@ async def test_update_entity_relations(
@pytest.mark.asyncio
async def test_two_pass_sync_flow(
knowledge_sync_service: KnowledgeSyncService, test_markdown: EntityMarkdown
knowledge_sync_service: EntitySyncService, test_markdown: EntityMarkdown
):
"""Test complete two-pass sync flow."""
# Create target entities first
@@ -179,7 +179,7 @@ async def test_two_pass_sync_flow(
await knowledge_sync_service.entity_repository.add(another_entity)
# First pass - create without relations
entity = await knowledge_sync_service.create_entity_and_observations("test.md", test_markdown)
entity = await knowledge_sync_service.create_entity_from_markdown("test.md", test_markdown)
assert len(entity.relations) == 0
assert entity.checksum is None