add search to sync flow

This commit is contained in:
phernandez
2025-01-04 21:19:42 -06:00
parent dbe9cea261
commit e35dbab156
7 changed files with 52 additions and 40 deletions
+42 -37
View File
@@ -25,6 +25,7 @@ from basic_memory.repository import (
ObservationRepository,
RelationRepository,
)
from basic_memory.repository.search_repository import SearchRepository
from basic_memory.services import (
DocumentService,
EntityService,
@@ -32,6 +33,7 @@ from basic_memory.services import (
RelationService,
FileService,
)
from basic_memory.services.search_service import SearchService
from basic_memory.sync import SyncService, FileChangeScanner, KnowledgeSyncService
from basic_memory.sync.utils import SyncReport
from basic_memory.utils.file_utils import ParseError
@@ -45,6 +47,46 @@ class ValidationIssue:
error: str
async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
"""Get sync service instance with all dependencies."""
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
engine,
session_maker,
):
# Initialize repositories
document_repository = DocumentRepository(session_maker)
entity_repository = EntityRepository(session_maker)
observation_repository = ObservationRepository(session_maker)
relation_repository = RelationRepository(session_maker)
search_repository = SearchRepository(session_maker)
# Initialize scanner
file_change_scanner = FileChangeScanner(document_repository, entity_repository)
# Initialize services
document_service = DocumentService(document_repository, config.documents_dir, FileService())
entity_service = EntityService(entity_repository)
observation_service = ObservationService(observation_repository)
relation_service = RelationService(relation_repository)
knowledge_sync_service = KnowledgeSyncService(
entity_service, observation_service, relation_service
)
knowledge_parser = KnowledgeParser()
search_service = SearchService(search_repository, document_service, entity_service)
# Create sync service
sync_service = SyncService(
scanner=file_change_scanner,
document_service=document_service,
knowledge_sync_service=knowledge_sync_service,
knowledge_parser=knowledge_parser,
search_service=search_service,
)
return sync_service
def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]:
"""Group validation issues by directory."""
grouped = defaultdict(list)
@@ -169,43 +211,6 @@ def display_detailed_sync_results(docs: SyncReport, knowledge: SyncReport):
console.print(knowledge_tree)
async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
"""Get sync service instance with all dependencies."""
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
engine,
session_maker,
):
# Initialize repositories
document_repository = DocumentRepository(session_maker)
entity_repository = EntityRepository(session_maker)
observation_repository = ObservationRepository(session_maker)
relation_repository = RelationRepository(session_maker)
# Initialize scanner
file_change_scanner = FileChangeScanner(document_repository, entity_repository)
# Initialize services
document_service = DocumentService(document_repository, config.documents_dir, FileService())
entity_service = EntityService(entity_repository)
observation_service = ObservationService(observation_repository)
relation_service = RelationService(relation_repository)
knowledge_sync_service = KnowledgeSyncService(
entity_service, observation_service, relation_service
)
knowledge_parser = KnowledgeParser()
# Create sync service
sync_service = SyncService(
scanner=file_change_scanner,
document_service=document_service,
knowledge_sync_service=knowledge_sync_service,
knowledge_parser=knowledge_parser,
)
return sync_service
async def validate_knowledge_files(
sync_service: SyncService, directory: Path
) -> List[ValidationIssue]:
+10 -3
View File
@@ -7,6 +7,7 @@ from loguru import logger
from basic_memory.config import ProjectConfig
from basic_memory.markdown import KnowledgeParser
from basic_memory.services import DocumentService
from basic_memory.services.search_service import SearchService
from basic_memory.sync import FileChangeScanner
from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService
from basic_memory.sync.utils import SyncReport
@@ -26,11 +27,13 @@ class SyncService:
document_service: DocumentService,
knowledge_sync_service: KnowledgeSyncService,
knowledge_parser: KnowledgeParser,
search_service: SearchService,
):
self.scanner = scanner
self.document_service = document_service
self.knowledge_sync_service = knowledge_sync_service
self.knowledge_parser = knowledge_parser
self.search_service = search_service
async def sync_documents(self, directory: Path) -> SyncReport:
"""Sync document files with database."""
@@ -47,12 +50,14 @@ class SyncService:
content = (directory / path).read_text()
if path in changes.new:
logger.debug(f"Creating new document: {path}")
await self.document_service.create_document(path_id=path, content=content)
document = await self.document_service.create_document(path_id=path, content=content)
else:
logger.debug(f"Updating document: {path}")
await self.document_service.update_document_by_path_id(
document = await self.document_service.update_document_by_path_id(
path_id=path, content=content
)
# add to search index
await self.search_service.index_document(document, content)
return changes
async def sync_knowledge(self, directory: Path) -> SyncReport:
@@ -89,9 +94,11 @@ class SyncService:
# Second pass: Process relations
for file_path, entity_markdown in parsed_entities.items():
logger.debug(f"Updating relations for: {file_path}")
await self.knowledge_sync_service.update_entity_relations(
entity = await self.knowledge_sync_service.update_entity_relations(
entity_markdown, checksum=changes.checksums[file_path]
)
# add to search index
await self.search_service.index_entity(entity)
return changes