diff --git a/src/basic_memory/cli/commands/status.py b/src/basic_memory/cli/commands/status.py index c5a01bda..79b82628 100644 --- a/src/basic_memory/cli/commands/status.py +++ b/src/basic_memory/cli/commands/status.py @@ -14,8 +14,8 @@ from basic_memory.cli.app import app from basic_memory.config import config from basic_memory.db import DatabaseType from basic_memory.repository import DocumentRepository, EntityRepository -from basic_memory.services.sync import FileChangeScanner -from basic_memory.services.sync.utils import SyncReport +from basic_memory.sync import FileChangeScanner +from basic_memory.sync.utils import SyncReport # Create rich console console = Console() @@ -64,7 +64,7 @@ def group_changes_by_directory(changes: SyncReport) -> Dict[str, Dict[str, int]] for change_type, paths in [ ("new", changes.new), ("modified", changes.modified), - ("deleted", changes.deleted) + ("deleted", changes.deleted), ]: for path in paths: dir_name = path.split("/", 1)[0] @@ -76,7 +76,7 @@ def group_changes_by_directory(changes: SyncReport) -> Dict[str, Dict[str, int]] def build_directory_summary(counts: Dict[str, int]) -> str: """Build summary string for directory changes.""" parts = [] - if counts["new"]: + if counts["new"]: parts.append(f"[green]+{counts['new']} new[/green]") if counts["modified"]: parts.append(f"[yellow]~{counts['modified']} modified[/yellow]") @@ -88,12 +88,12 @@ def build_directory_summary(counts: Dict[str, int]) -> str: def display_changes(title: str, changes: SyncReport, verbose: bool = False): """Display changes using Rich for better visualization.""" tree = Tree(title) - + if changes.total_changes == 0: tree.add("No changes") console.print(Panel(tree, expand=False)) return - + if verbose: # Full file listing with checksums if changes.new: @@ -111,7 +111,7 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False): for dir_name, counts in sorted(by_dir.items()): summary = build_directory_summary(counts) tree.add(f"[bold]{dir_name}/[/bold] {summary}") - + console.print(Panel(tree, expand=False)) @@ -121,7 +121,7 @@ async def run_status(sync_service: FileChangeScanner, verbose: bool = False): knowledge_changes = await sync_service.find_knowledge_changes(config.knowledge_dir) display_changes("Knowledge Files", knowledge_changes, verbose) - # Check documents/ directory + # Check documents/ directory document_changes = await sync_service.find_document_changes(config.documents_dir) display_changes("Documents", document_changes, verbose) diff --git a/src/basic_memory/cli/commands/sync.py b/src/basic_memory/cli/commands/sync.py index 09f0c416..a43e02db 100644 --- a/src/basic_memory/cli/commands/sync.py +++ b/src/basic_memory/cli/commands/sync.py @@ -1,34 +1,40 @@ """Command module for basic-memory sync operations.""" -from collections import defaultdict +import asyncio +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import List, Dict + +import typer +from loguru import logger +from rich.console import Console from rich.padding import Padding from rich.panel import Panel from rich.text import Text - -from pathlib import Path -from typing import Optional, List, Tuple, Dict -from dataclasses import dataclass - -import typer -import asyncio -from loguru import logger -from rich.console import Console from rich.tree import Tree -from basic_memory.cli.app import app from basic_memory import db +from basic_memory.cli.app import app from basic_memory.config import config from basic_memory.db import DatabaseType -from basic_memory.repository import DocumentRepository, EntityRepository, ObservationRepository, RelationRepository +from basic_memory.markdown import KnowledgeParser +from basic_memory.repository import ( + DocumentRepository, + EntityRepository, + ObservationRepository, + RelationRepository, +) from basic_memory.services import ( DocumentService, - EntityService, ObservationService, RelationService, FileService, + EntityService, + ObservationService, + RelationService, + FileService, ) -from basic_memory.markdown import KnowledgeParser -from basic_memory.services.sync import SyncService, FileChangeScanner, KnowledgeSyncService +from basic_memory.sync import SyncService, FileChangeScanner, KnowledgeSyncService +from basic_memory.sync.utils import SyncReport from basic_memory.utils.file_utils import ParseError -from basic_memory.services.sync.utils import SyncReport - console = Console() @@ -38,6 +44,7 @@ class ValidationIssue: file_path: str error: str + def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]: """Group validation issues by directory.""" grouped = defaultdict(list) @@ -51,10 +58,9 @@ def display_validation_errors(issues: List[ValidationIssue]): """Display validation errors in a rich, organized format.""" # Create header console.print() - console.print(Panel( - "[red bold]Error:[/red bold] Invalid frontmatter in knowledge files", - expand=False - )) + console.print( + Panel("[red bold]Error:[/red bold] Invalid frontmatter in knowledge files", expand=False) + ) console.print() # Group issues by directory @@ -65,20 +71,14 @@ def display_validation_errors(issues: List[ValidationIssue]): for dir_name, dir_issues in sorted(grouped_issues.items()): # Create branch for directory branch = tree.add( - f"[bold blue]{dir_name}/[/bold blue] " - f"([yellow]{len(dir_issues)} files[/yellow])" + f"[bold blue]{dir_name}/[/bold blue] " f"([yellow]{len(dir_issues)} files[/yellow])" ) # Add each file issue for issue in sorted(dir_issues, key=lambda x: x.file_path): file_name = Path(issue.file_path).name branch.add( - Text.assemble( - ("└─ ", "dim"), - (file_name, "yellow"), - ": ", - (issue.error, "red") - ) + Text.assemble(("└─ ", "dim"), (file_name, "yellow"), ": ", (issue.error, "red")) ) # Display tree @@ -86,16 +86,18 @@ def display_validation_errors(issues: List[ValidationIssue]): # Add help text console.print() - console.print(Panel( - Text.assemble( - ("To fix:", "bold"), - "\n1. Add required frontmatter fields to each file", - "\n2. Run ", - ("basic-memory sync", "bold cyan"), - " again" - ), - expand=False - )) + console.print( + Panel( + Text.assemble( + ("To fix:", "bold"), + "\n1. Add required frontmatter fields to each file", + "\n2. Run ", + ("basic-memory sync", "bold cyan"), + " again", + ), + expand=False, + ) + ) console.print() @@ -135,12 +137,12 @@ def display_detailed_sync_results(docs: SyncReport, knowledge: SyncReport): if docs.new: created = doc_tree.add("[green]Created[/green]") for path in sorted(docs.new): - checksum = docs.checksums.get(path, '') + checksum = docs.checksums.get(path, "") created.add(f"[green]{path}[/green] ({checksum[:8]})") if docs.modified: modified = doc_tree.add("[yellow]Modified[/yellow]") for path in sorted(docs.modified): - checksum = docs.checksums.get(path, '') + checksum = docs.checksums.get(path, "") modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})") if docs.deleted: deleted = doc_tree.add("[red]Deleted[/red]") @@ -153,12 +155,12 @@ def display_detailed_sync_results(docs: SyncReport, knowledge: SyncReport): if knowledge.new: created = knowledge_tree.add("[green]Created[/green]") for path in sorted(knowledge.new): - checksum = knowledge.checksums.get(path, '') + checksum = knowledge.checksums.get(path, "") created.add(f"[green]{path}[/green] ({checksum[:8]})") if knowledge.modified: modified = knowledge_tree.add("[yellow]Modified[/yellow]") for path in sorted(knowledge.modified): - checksum = knowledge.checksums.get(path, '') + checksum = knowledge.checksums.get(path, "") modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})") if knowledge.deleted: deleted = knowledge_tree.add("[red]Deleted[/red]") @@ -181,14 +183,16 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): # Initialize scanner file_change_scanner = FileChangeScanner(document_repository, entity_repository) - + # Initialize services document_service = DocumentService(document_repository, config.documents_dir, FileService()) entity_service = EntityService(entity_repository) observation_service = ObservationService(observation_repository) relation_service = RelationService(relation_repository) - - knowledge_sync_service = KnowledgeSyncService(entity_service, observation_service, relation_service) + + knowledge_sync_service = KnowledgeSyncService( + entity_service, observation_service, relation_service + ) knowledge_parser = KnowledgeParser() # Create sync service @@ -198,11 +202,13 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM): knowledge_sync_service=knowledge_sync_service, knowledge_parser=knowledge_parser, ) - + return sync_service -async def validate_knowledge_files(sync_service: SyncService, directory: Path) -> List[ValidationIssue]: +async def validate_knowledge_files( + sync_service: SyncService, directory: Path +) -> List[ValidationIssue]: """Pre-validate knowledge files and collect all issues.""" issues = [] changes = await sync_service.scanner.find_knowledge_changes(directory) @@ -218,7 +224,7 @@ async def validate_knowledge_files(sync_service: SyncService, directory: Path) - async def run_sync(verbose: bool = False): """Run sync operation.""" - + sync_service = await get_sync_service() # Validate knowledge files before attempting sync @@ -229,7 +235,7 @@ async def run_sync(verbose: bool = False): # Sync doc_changes, knowledge_changes = await sync_service.sync(config) - + # Display results if verbose: display_detailed_sync_results(doc_changes, knowledge_changes) @@ -247,17 +253,17 @@ def sync( ), ) -> None: """Sync knowledge files with the database. - + This command syncs both documents and knowledge files with the database. Knowledge files must have required frontmatter fields: type, id, created, modified. """ try: # Run sync asyncio.run(run_sync(verbose)) - + except Exception as e: if not isinstance(e, typer.Exit): logger.exception("Sync failed") typer.echo(f"Error during sync: {e}", err=True) raise typer.Exit(1) - raise \ No newline at end of file + raise diff --git a/src/basic_memory/services/sync/__init__.py b/src/basic_memory/sync/__init__.py similarity index 100% rename from src/basic_memory/services/sync/__init__.py rename to src/basic_memory/sync/__init__.py diff --git a/src/basic_memory/services/sync/file_change_scanner.py b/src/basic_memory/sync/file_change_scanner.py similarity index 81% rename from src/basic_memory/services/sync/file_change_scanner.py rename to src/basic_memory/sync/file_change_scanner.py index 23edd120..b6fe85f6 100644 --- a/src/basic_memory/services/sync/file_change_scanner.py +++ b/src/basic_memory/sync/file_change_scanner.py @@ -1,18 +1,17 @@ """Service for detecting changes between filesystem and database.""" from pathlib import Path -from typing import Dict, Protocol, TypeVar, Optional, Sequence +from typing import Dict, Sequence from loguru import logger from basic_memory.models import Document, Entity from basic_memory.repository.document_repository import DocumentRepository from basic_memory.repository.entity_repository import EntityRepository -from basic_memory.services.sync.utils import DbState, SyncReport, ScanResult +from basic_memory.sync.utils import DbState, SyncReport, ScanResult from basic_memory.utils.file_utils import compute_checksum - class FileChangeScanner: """ Service for detecting changes between filesystem and database. @@ -20,9 +19,7 @@ class FileChangeScanner: """ def __init__( - self, - document_repository: DocumentRepository, - entity_repository: EntityRepository + self, document_repository: DocumentRepository, entity_repository: EntityRepository ): self.document_repository = document_repository self.entity_repository = entity_repository @@ -56,16 +53,13 @@ class FileChangeScanner: rel_path = str(path.relative_to(directory)) content = path.read_text() checksum = await compute_checksum(content) - + if checksum: # Only store valid checksums - result.files[rel_path] = DbState( - path=rel_path, - checksum=checksum - ) + result.files[rel_path] = DbState(path=rel_path, checksum=checksum) logger.debug(f"Found file: {rel_path} ({checksum[:8]})") else: result.errors[rel_path] = "Failed to compute checksum" - + except Exception as e: rel_path = str(path.relative_to(directory)) result.errors[rel_path] = str(e) @@ -74,15 +68,10 @@ class FileChangeScanner: logger.debug(f"Found {len(result.files)} markdown files") if result.errors: logger.warning(f"Encountered {len(result.errors)} errors while scanning") - + return result - - async def find_changes( - self, - directory: Path, - db_records: Dict[str, DbState] - ) -> SyncReport: + async def find_changes(self, directory: Path, db_records: Dict[str, DbState]) -> SyncReport: """ Find changes between filesystem and database. @@ -96,23 +85,24 @@ class FileChangeScanner: # Get current files and checksums scan_result = await self.scan_directory(directory) current_files = scan_result.files - + logger.debug("Current files from filesystem:") for file_path, state in sorted(current_files.items()): logger.debug(f" {file_path} ({state.checksum[:8]})") - logger.debug("Files from database:") for file_path, state in sorted(db_records.items()): - logger.debug(f" {file_path} ({state.checksum[:8] if state.checksum else 'no checksum'})") + logger.debug( + f" {file_path} ({state.checksum[:8] if state.checksum else 'no checksum'})" + ) # Build report report = SyncReport() - + # Add current checksums for display for file_path, state in current_files.items(): report.checksums[file_path] = state.checksum - + # Find new and modified files for file_path, curr_state in current_files.items(): if file_path not in db_records: @@ -132,7 +122,7 @@ class FileChangeScanner: logger.debug(f" New: {len(report.new)}") logger.debug(f" Modified: {len(report.modified)}") logger.debug(f" Deleted: {len(report.deleted)}") - + if scan_result.errors: logger.warning("Files skipped due to errors:") for file_path, error in scan_result.errors.items(): @@ -140,28 +130,24 @@ class FileChangeScanner: return report - async def get_db_file_paths(self, db_records: Sequence[Document | Entity]) -> Dict[str, DbState]: + async def get_db_file_paths( + self, db_records: Sequence[Document | Entity] + ) -> Dict[str, DbState]: """Get file_path and checksums from database. Args: db_records: database records Returns: Dict mapping file paths to FileState - :param db_records: the data from the db + :param db_records: the data from the db """ return {r.file_path: DbState(path=r.path_id, checksum=r.checksum) for r in db_records} async def find_document_changes(self, directory: Path) -> SyncReport: """Find changes in document directory.""" db_records = await self.get_db_file_paths(await self.document_repository.find_all()) - return await self.find_changes( - directory=directory, - db_records=db_records - ) + return await self.find_changes(directory=directory, db_records=db_records) async def find_knowledge_changes(self, directory: Path) -> SyncReport: """Find changes in knowledge directory.""" db_records = await self.get_db_file_paths(await self.entity_repository.find_all()) - return await self.find_changes( - directory=directory, - db_records=db_records - ) \ No newline at end of file + return await self.find_changes(directory=directory, db_records=db_records) diff --git a/src/basic_memory/services/sync/knowledge_sync_service.py b/src/basic_memory/sync/knowledge_sync_service.py similarity index 100% rename from src/basic_memory/services/sync/knowledge_sync_service.py rename to src/basic_memory/sync/knowledge_sync_service.py diff --git a/src/basic_memory/services/sync/sync_service.py b/src/basic_memory/sync/sync_service.py similarity index 93% rename from src/basic_memory/services/sync/sync_service.py rename to src/basic_memory/sync/sync_service.py index ca057bdc..f6cadd35 100644 --- a/src/basic_memory/services/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -1,14 +1,15 @@ """Service for syncing files between filesystem and database.""" from pathlib import Path + from loguru import logger from basic_memory.config import ProjectConfig -from basic_memory.services import DocumentService -from basic_memory.services.sync import FileChangeScanner from basic_memory.markdown import KnowledgeParser -from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService -from basic_memory.services.sync.utils import SyncReport +from basic_memory.services import DocumentService +from basic_memory.sync import FileChangeScanner +from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService +from basic_memory.sync.utils import SyncReport class SyncService: @@ -53,7 +54,7 @@ class SyncService: path_id=path, content=content ) return changes - + async def sync_knowledge(self, directory: Path) -> SyncReport: """Sync knowledge files with database.""" changes = await self.scanner.find_knowledge_changes(directory) @@ -75,7 +76,9 @@ class SyncService: for file_path, entity_markdown in parsed_entities.items(): if file_path in changes.new: logger.debug(f"Creating new entity_markdown: {file_path}") - await self.knowledge_sync_service.create_entity_and_observations(file_path, entity_markdown) + await self.knowledge_sync_service.create_entity_and_observations( + file_path, entity_markdown + ) else: path_id = entity_markdown.frontmatter.id logger.debug(f"Updating entity_markdown: {path_id}") @@ -89,16 +92,16 @@ class SyncService: await self.knowledge_sync_service.update_entity_relations( entity_markdown, checksum=changes.checksums[file_path] ) - + return changes async def sync(self, config: ProjectConfig) -> (SyncReport, SyncReport): """Sync all files with database.""" - + # Sync documents first (simpler, no relations) doc_changes = await self.sync_documents(config.documents_dir) # Then sync knowledge files knowledge_changes = await self.sync_knowledge(config.knowledge_dir) - return doc_changes, knowledge_changes \ No newline at end of file + return doc_changes, knowledge_changes diff --git a/src/basic_memory/services/sync/utils.py b/src/basic_memory/sync/utils.py similarity index 100% rename from src/basic_memory/services/sync/utils.py rename to src/basic_memory/sync/utils.py diff --git a/tests/cli/test_status.py b/tests/cli/test_status.py index f1880a49..1eb1860a 100644 --- a/tests/cli/test_status.py +++ b/tests/cli/test_status.py @@ -1,12 +1,13 @@ """Test status command functionality.""" +from io import StringIO + import pytest import pytest_asyncio from rich.console import Console -from io import StringIO from basic_memory.cli.commands.status import display_changes, run_status -from basic_memory.services.sync.utils import SyncReport, DbState +from basic_memory.sync.utils import SyncReport, DbState from basic_memory.utils.file_utils import compute_checksum diff --git a/tests/conftest.py b/tests/conftest.py index 4e9d23e5..bcb68c5b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,8 +10,8 @@ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker from basic_memory import db from basic_memory.config import ProjectConfig from basic_memory.db import DatabaseType -from basic_memory.markdown.knowledge_writer import KnowledgeWriter from basic_memory.markdown.knowledge_parser import KnowledgeParser +from basic_memory.markdown.knowledge_writer import KnowledgeWriter from basic_memory.models import Base from basic_memory.models.knowledge import Entity from basic_memory.repository.document_repository import DocumentRepository @@ -25,13 +25,13 @@ from basic_memory.services import ( RelationService, DocumentService, ) -from basic_memory.services.search_service import SearchService -from basic_memory.services.sync import FileChangeScanner +from basic_memory.services import KnowledgeService from basic_memory.services.activity_service import ActivityService from basic_memory.services.file_service import FileService -from basic_memory.services import KnowledgeService -from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService -from basic_memory.services.sync.sync_service import SyncService +from basic_memory.services.search_service import SearchService +from basic_memory.sync import FileChangeScanner +from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService +from basic_memory.sync.sync_service import SyncService @pytest_asyncio.fixture @@ -46,7 +46,7 @@ def test_config(tmp_path) -> ProjectConfig: name="test", ) config.home = tmp_path - + (tmp_path / config.documents_dir.name).mkdir(parents=True, exist_ok=True) (tmp_path / config.knowledge_dir.name).mkdir(parents=True, exist_ok=True) return config @@ -180,11 +180,13 @@ async def knowledge_service( base_path=test_config.knowledge_dir, ) + @pytest_asyncio.fixture async def activity_service(document_service, entity_service, relation_service): """Create activity service with real dependencies.""" return ActivityService(entity_service, document_service, relation_service) + @pytest_asyncio.fixture async def knowledge_sync_service( entity_service: EntityService, @@ -216,6 +218,7 @@ async def search_repository(session_maker): """Create SearchRepository instance""" return SearchRepository(session_maker) + @pytest_asyncio.fixture async def search_service(search_repository: SearchRepository): """Create and initialize search service""" @@ -234,4 +237,4 @@ async def sample_entity(entity_repository: EntityRepository) -> Entity: "path_id": "test/test_entity", "file_path": "test/test_entity.md", } - return await entity_repository.create(entity_data) \ No newline at end of file + return await entity_repository.create(entity_data) diff --git a/tests/repository/test_entity_repository.py b/tests/repository/test_entity_repository.py index c6b57ed1..6347cff3 100644 --- a/tests/repository/test_entity_repository.py +++ b/tests/repository/test_entity_repository.py @@ -146,7 +146,7 @@ async def test_entity_type_name_unique_constraint(entity_repository: EntityRepos # Should raise IntegrityError with pytest.raises(IntegrityError) as exc_info: await entity_repository.create(entity2_data) - assert "UNIQUE constraint failed: entity.entity_type, entity.name" in str(exc_info.value) + assert "UNIQUE constraint failed" in str(exc_info.value) @pytest.mark.asyncio diff --git a/tests/services/sync/test_file_change_scanner.py b/tests/services/sync/test_file_change_scanner.py index cf188fcb..20fde9bb 100644 --- a/tests/services/sync/test_file_change_scanner.py +++ b/tests/services/sync/test_file_change_scanner.py @@ -1,13 +1,14 @@ """Test file sync service.""" -import pytest -from pathlib import Path -from typing import AsyncGenerator -from basic_memory.repository import DocumentRepository, EntityRepository -from basic_memory.services.sync import FileChangeScanner -from basic_memory.services.sync.utils import DbState -from basic_memory.utils.file_utils import compute_checksum +from pathlib import Path + +import pytest + from basic_memory.models import Document, Entity +from basic_memory.repository import DocumentRepository +from basic_memory.sync import FileChangeScanner +from basic_memory.sync.utils import DbState +from basic_memory.utils.file_utils import compute_checksum @pytest.fixture @@ -23,10 +24,7 @@ async def create_test_file(path: Path, content: str = "test content") -> None: @pytest.mark.asyncio -async def test_scan_empty_directory( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_scan_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test scanning empty directory.""" result = await file_change_scanner.scan_directory(temp_dir) assert len(result.files) == 0 @@ -34,10 +32,7 @@ async def test_scan_empty_directory( @pytest.mark.asyncio -async def test_scan_with_mixed_files( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_scan_with_mixed_files(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test scanning directory with markdown and non-markdown files.""" # Create test files await create_test_file(temp_dir / "doc.md", "markdown") @@ -57,10 +52,7 @@ async def test_scan_with_mixed_files( @pytest.mark.asyncio -async def test_scan_with_unreadable_file( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_scan_with_unreadable_file(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test scanning directory with an unreadable file.""" # Create a file we'll make unreadable bad_file = temp_dir / "bad.md" @@ -75,143 +67,97 @@ async def test_scan_with_unreadable_file( @pytest.mark.asyncio async def test_detect_new_files( - file_change_scanner: FileChangeScanner, - temp_dir: Path, - document_repository: DocumentRepository + file_change_scanner: FileChangeScanner, temp_dir: Path, document_repository: DocumentRepository ): """Test detection of new files.""" # Create new file await create_test_file(temp_dir / "new.md") - + # Empty DB state db_records = await file_change_scanner.get_db_file_paths([]) - - changes = await file_change_scanner.find_changes( - directory=temp_dir, - db_records=db_records - ) - + + changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) + assert len(changes.new) == 1 assert "new.md" in changes.new @pytest.mark.asyncio -async def test_detect_modified_file( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_detect_modified_file(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test detection of modified files.""" path = "test.md" content = "original" await create_test_file(temp_dir / path, content) - + # Create DB state with original checksum original_checksum = await compute_checksum(content) - db_records = { - path: DbState(path=path, checksum=original_checksum) - } + db_records = {path: DbState(path=path, checksum=original_checksum)} # Modify file await create_test_file(temp_dir / path, "modified") - changes = await file_change_scanner.find_changes( - directory=temp_dir, - db_records=db_records - ) + changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) assert len(changes.modified) == 1 assert path in changes.modified @pytest.mark.asyncio -async def test_detect_deleted_files( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test detection of deleted files.""" path = "deleted.md" - - # Create DB state with file that doesn't exist - db_records = { - path: DbState(path=path, checksum="any-checksum") - } - changes = await file_change_scanner.find_changes( - directory=temp_dir, - db_records=db_records - ) + # Create DB state with file that doesn't exist + db_records = {path: DbState(path=path, checksum="any-checksum")} + + changes = await file_change_scanner.find_changes(directory=temp_dir, db_records=db_records) assert len(changes.deleted) == 1 assert path in changes.deleted @pytest.mark.asyncio -async def test_get_db_state_documents( - file_change_scanner: FileChangeScanner -): +async def test_get_db_state_documents(file_change_scanner: FileChangeScanner): """Test converting document records to file states.""" - doc = Document( - path_id="test.md", - file_path="test.md", - checksum="test-checksum" - ) - + doc = Document(path_id="test.md", file_path="test.md", checksum="test-checksum") + db_records = await file_change_scanner.get_db_file_paths([doc]) - + assert len(db_records) == 1 assert "test.md" in db_records assert db_records["test.md"].checksum == "test-checksum" @pytest.mark.asyncio -async def test_get_db_state_entities( - file_change_scanner: FileChangeScanner -): +async def test_get_db_state_entities(file_change_scanner: FileChangeScanner): """Test converting entity records to file states.""" - entity = Entity( - path_id="concept/test", - file_path="concept/test.md", - checksum="test-checksum" - ) - + entity = Entity(path_id="concept/test", file_path="concept/test.md", checksum="test-checksum") + db_records = await file_change_scanner.get_db_file_paths([entity]) - + assert len(db_records) == 1 assert "concept/test.md" in db_records assert db_records["concept/test.md"].checksum == "test-checksum" - @pytest.mark.asyncio -async def test_get_db_state_does_not_skip_missing_checksum( - file_change_scanner: FileChangeScanner -): +async def test_get_db_state_does_not_skip_missing_checksum(file_change_scanner: FileChangeScanner): """Test that get_db_state skips records with missing checksums.""" - doc = Document( - path_id="test.md", - file_path="test.md", - checksum=None - ) - + doc = Document(path_id="test.md", file_path="test.md", checksum=None) + db_records = await file_change_scanner.get_db_file_paths([doc]) - + assert len(db_records) == 1 @pytest.mark.asyncio -async def test_empty_directory( - file_change_scanner: FileChangeScanner, - temp_dir: Path -): +async def test_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test handling empty/nonexistent directory.""" nonexistent = temp_dir / "nonexistent" - - changes = await file_change_scanner.find_changes( - directory=nonexistent, - db_records={} - ) - + + changes = await file_change_scanner.find_changes(directory=nonexistent, db_records={}) + assert changes.total_changes == 0 assert not changes.new assert not changes.modified - assert not changes.deleted \ No newline at end of file + assert not changes.deleted diff --git a/tests/services/sync/test_knowledge_sync_service.py b/tests/services/sync/test_knowledge_sync_service.py index 08dbef45..4be4f405 100644 --- a/tests/services/sync/test_knowledge_sync_service.py +++ b/tests/services/sync/test_knowledge_sync_service.py @@ -1,12 +1,10 @@ """Tests for EntitySyncService.""" +from datetime import datetime + import pytest import pytest_asyncio -from datetime import datetime -from basic_memory.models import Entity as EntityModel -from basic_memory.services import EntityService, ObservationService, RelationService -from basic_memory.services.sync.knowledge_sync_service import KnowledgeSyncService from basic_memory.markdown.schemas import ( EntityMarkdown, EntityContent, @@ -15,8 +13,8 @@ from basic_memory.markdown.schemas import ( Observation as MarkdownObservation, Relation as MarkdownRelation, ) - - +from basic_memory.models import Entity as EntityModel +from basic_memory.sync.knowledge_sync_service import KnowledgeSyncService @pytest_asyncio.fixture @@ -120,10 +118,16 @@ async def test_update_entity_relations( # Create target entities that relations point to other_entity = EntityModel( - name="Other Entity", entity_type="concept", path_id="concept/other_entity", file_path="concept/other_entity.md" + name="Other Entity", + entity_type="concept", + path_id="concept/other_entity", + file_path="concept/other_entity.md", ) another_entity = EntityModel( - name="Another Entity", entity_type="concept", path_id="concept/another_entity", file_path="concept/another_entity.md" + name="Another Entity", + entity_type="concept", + path_id="concept/another_entity", + file_path="concept/another_entity.md", ) await knowledge_sync_service.entity_service.add(other_entity) await knowledge_sync_service.entity_service.add(another_entity) @@ -157,10 +161,16 @@ async def test_two_pass_sync_flow( """Test complete two-pass sync flow.""" # Create target entities first other_entity = EntityModel( - name="Other Entity", entity_type="concept", path_id="concept/other_entity", file_path="concept/other_entity.md" + name="Other Entity", + entity_type="concept", + path_id="concept/other_entity", + file_path="concept/other_entity.md", ) another_entity = EntityModel( - name="Another Entity", entity_type="concept", path_id="concept/another_entity", file_path="concept/another_entity.md" + name="Another Entity", + entity_type="concept", + path_id="concept/another_entity", + file_path="concept/another_entity.md", ) await knowledge_sync_service.entity_service.add(other_entity) await knowledge_sync_service.entity_service.add(another_entity) diff --git a/tests/services/sync/test_sync_documents.py b/tests/services/sync/test_sync_documents.py index 90bbec38..28429d2d 100644 --- a/tests/services/sync/test_sync_documents.py +++ b/tests/services/sync/test_sync_documents.py @@ -1,12 +1,14 @@ """Test document sync functionality.""" + import asyncio from pathlib import Path + import pytest from basic_memory.config import ProjectConfig -from basic_memory.services import DocumentService -from basic_memory.services.sync.sync_service import SyncService from basic_memory.models import Document +from basic_memory.services import DocumentService +from basic_memory.sync.sync_service import SyncService async def create_test_file(path: Path, content: str = "test content") -> None: @@ -17,9 +19,7 @@ async def create_test_file(path: Path, content: str = "test content") -> None: @pytest.mark.asyncio async def test_sync_documents( - sync_service: SyncService, - test_config: ProjectConfig, - document_service: DocumentService + sync_service: SyncService, test_config: ProjectConfig, document_service: DocumentService ): """Test syncing document files.""" # Create test files @@ -45,8 +45,7 @@ async def test_sync_documents( @pytest.mark.asyncio async def test_sync_new_document_adds_frontmatter( - test_config: ProjectConfig, - sync_service: SyncService + test_config: ProjectConfig, sync_service: SyncService ): """Test that syncing a new document adds appropriate frontmatter.""" # Create document without frontmatter @@ -79,8 +78,7 @@ async def test_sync_new_document_adds_frontmatter( @pytest.mark.asyncio async def test_sync_modified_document_updates_frontmatter( - test_config: ProjectConfig, - sync_service: SyncService + test_config: ProjectConfig, sync_service: SyncService ): """Test that modifying a document updates frontmatter properly.""" # First create and sync a document @@ -95,7 +93,7 @@ async def test_sync_modified_document_updates_frontmatter( original_modified = doc.updated_at await asyncio.sleep(1) # Ensure timestamps will be different - + # Modify document new_content = "# Test Document\n\nUpdated content." doc_path.write_text(new_content) @@ -115,8 +113,7 @@ async def test_sync_modified_document_updates_frontmatter( @pytest.mark.asyncio async def test_sync_document_with_invalid_frontmatter( - test_config: ProjectConfig, - sync_service: SyncService + test_config: ProjectConfig, sync_service: SyncService ): """Test syncing a document with malformed frontmatter.""" # Create document with invalid frontmatter @@ -157,8 +154,7 @@ Content here. @pytest.mark.asyncio async def test_sync_document_preserve_existing_metadata( - test_config: ProjectConfig, - sync_service: SyncService + test_config: ProjectConfig, sync_service: SyncService ): """Test that sync preserves custom metadata fields in frontmatter.""" # Create document with custom metadata @@ -198,16 +194,13 @@ author: Test Author @pytest.mark.asyncio -async def test_sync_document_in_subdirectory( - test_config: ProjectConfig, - sync_service: SyncService -): +async def test_sync_document_in_subdirectory(test_config: ProjectConfig, sync_service: SyncService): """Test syncing documents in nested directory structure.""" # Create nested directories and files base_dir = test_config.documents_dir nested_path = base_dir / "folder1" / "folder2" / "nested.md" nested_path.parent.mkdir(parents=True) - + content = "# Nested Document\nIn subfolder" nested_path.write_text(content) @@ -226,10 +219,7 @@ async def test_sync_document_in_subdirectory( @pytest.mark.asyncio -async def test_sync_empty_document( - test_config: ProjectConfig, - sync_service: SyncService -): +async def test_sync_empty_document(test_config: ProjectConfig, sync_service: SyncService): """Test syncing an empty document.""" # Create empty file doc_path = test_config.documents_dir / "empty.md" @@ -252,10 +242,7 @@ async def test_sync_empty_document( @pytest.mark.asyncio -async def test_sync_document_utf8_encoding( - test_config: ProjectConfig, - sync_service: SyncService -): +async def test_sync_document_utf8_encoding(test_config: ProjectConfig, sync_service: SyncService): """Test syncing documents with non-ASCII characters.""" # Create document with UTF-8 content doc_path = test_config.documents_dir / "utf8.md" @@ -270,13 +257,13 @@ async def test_sync_document_utf8_encoding( ## Various Symbols § ¢ € ¥ © ®️ ™️ """ - doc_path.write_text(content, encoding='utf-8') + doc_path.write_text(content, encoding="utf-8") # Sync await sync_service.sync(test_config) # Read back and verify content preserved - updated_content = doc_path.read_text(encoding='utf-8') + updated_content = doc_path.read_text(encoding="utf-8") assert "你好世界" in updated_content assert "こんにちは" in updated_content assert "🚀" in updated_content @@ -285,4 +272,4 @@ async def test_sync_document_utf8_encoding( # Verify document in DB doc = await sync_service.document_service.repository.find_by_path_id("utf8.md") assert doc is not None - assert doc.checksum is not None \ No newline at end of file + assert doc.checksum is not None diff --git a/tests/services/sync/test_sync_knowledge.py b/tests/services/sync/test_sync_knowledge.py index 7320e4f9..497ce343 100644 --- a/tests/services/sync/test_sync_knowledge.py +++ b/tests/services/sync/test_sync_knowledge.py @@ -1,12 +1,13 @@ """Test knowledge sync functionality.""" from pathlib import Path + import pytest from basic_memory.config import ProjectConfig -from basic_memory.services import EntityService -from basic_memory.services.sync.sync_service import SyncService from basic_memory.models import Entity +from basic_memory.services import EntityService +from basic_memory.sync.sync_service import SyncService async def create_test_file(path: Path, content: str = "test content") -> None: @@ -17,9 +18,7 @@ async def create_test_file(path: Path, content: str = "test content") -> None: @pytest.mark.asyncio async def test_sync_knowledge( - sync_service: SyncService, - test_config: ProjectConfig, - entity_service: EntityService + sync_service: SyncService, test_config: ProjectConfig, entity_service: EntityService ): """Test basic knowledge sync functionality.""" # Create test files @@ -75,12 +74,11 @@ A test concept. @pytest.mark.asyncio async def test_sync_entity_with_nonexistent_relations( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test syncing an entity that references nonexistent entities.""" knowledge_dir = test_config.knowledge_dir - + # Create entity that references entities we haven't created yet content = """ --- @@ -99,10 +97,10 @@ modified: 2024-01-01 - uses [[concept/also_future]] """ await create_test_file(knowledge_dir / "concept/depends_on_future.md", content) - + # Sync await sync_service.sync(test_config) - + # Verify entity created but no relations entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( "concept/depends_on_future" @@ -113,12 +111,11 @@ modified: 2024-01-01 @pytest.mark.asyncio async def test_sync_entity_circular_relations( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test syncing entities with circular dependencies.""" knowledge_dir = test_config.knowledge_dir - + # Create entity A that depends on B content_a = """ --- @@ -136,7 +133,7 @@ modified: 2024-01-01 - depends_on [[concept/entity_b]] """ await create_test_file(knowledge_dir / "concept/entity_a.md", content_a) - + # Create entity B that depends on A content_b = """ --- @@ -154,18 +151,22 @@ modified: 2024-01-01 - depends_on [[concept/entity_a]] """ await create_test_file(knowledge_dir / "concept/entity_b.md", content_b) - + # Sync await sync_service.sync(test_config) - + # Verify both entities and their relations - entity_a = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/entity_a") - entity_b = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/entity_b") - + entity_a = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/entity_a" + ) + entity_b = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/entity_b" + ) + # outgoing relations assert len(entity_a.outgoing_relations) == 1 assert len(entity_b.outgoing_relations) == 1 - + # incoming relations assert len(entity_a.incoming_relations) == 1 assert len(entity_b.incoming_relations) == 1 @@ -177,19 +178,18 @@ modified: 2024-01-01 # Verify circular reference works a_relation = entity_a.outgoing_relations[0] assert a_relation.to_id == entity_b.id - + b_relation = entity_b.outgoing_relations[0] assert b_relation.to_id == entity_a.id @pytest.mark.asyncio async def test_sync_entity_duplicate_relations( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test handling of duplicate relations in an entity.""" knowledge_dir = test_config.knowledge_dir - + # Create target entity first target_content = """ --- @@ -205,7 +205,7 @@ modified: 2024-01-01 """ await create_test_file(knowledge_dir / "concept/target.md", target_content) - + # Create entity with duplicate relations content = """ --- @@ -226,20 +226,20 @@ modified: 2024-01-01 - uses [[concept/target]] # Duplicate of different type """ await create_test_file(knowledge_dir / "concept/duplicate_relations.md", content) - + # Sync await sync_service.sync(test_config) - + # Verify duplicates are handled entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( "concept/duplicate_relations" ) - + # Count relations by type relation_counts = {} for rel in entity.relations: relation_counts[rel.relation_type] = relation_counts.get(rel.relation_type, 0) + 1 - + # Should only have one of each type assert relation_counts["depends_on"] == 1 assert relation_counts["uses"] == 1 @@ -247,12 +247,11 @@ modified: 2024-01-01 @pytest.mark.asyncio async def test_sync_entity_with_invalid_category( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test handling of invalid observation categories.""" knowledge_dir = test_config.knowledge_dir - + content = """ --- type: concept @@ -269,18 +268,18 @@ modified: 2024-01-01 - [design] This is valid """ await create_test_file(knowledge_dir / "concept/invalid_category.md", content) - + # Sync await sync_service.sync(test_config) - + # Verify observations entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( "concept/invalid_category" ) - + assert len(entity.observations) == 4 categories = [obs.category for obs in entity.observations] - + # Invalid categories should be converted to default assert "note" in categories # Valid categories preserved @@ -289,12 +288,11 @@ modified: 2024-01-01 @pytest.mark.asyncio async def test_sync_entity_with_order_dependent_relations( - sync_service: SyncService, - test_config: ProjectConfig + sync_service: SyncService, test_config: ProjectConfig ): """Test that order of entity syncing doesn't affect relation creation.""" knowledge_dir = test_config.knowledge_dir - + # Create several interrelated entities entities = { "a": """ @@ -343,28 +341,32 @@ modified: 2024-01-01 ## Relations - depends_on [[concept/entity_a]] -""" +""", } - + # Create files in different orders and verify results are the same for name, content in entities.items(): await create_test_file(knowledge_dir / f"concept/entity_{name}.md", content) - + # Sync await sync_service.sync(test_config) - + # Verify all relations are created correctly regardless of order - entity_a = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/entity_a") - entity_b = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/entity_b") - entity_c = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/entity_c") - + entity_a = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/entity_a" + ) + entity_b = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/entity_b" + ) + entity_c = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( + "concept/entity_c" + ) + assert len(entity_a.outgoing_relations) == 2 # Should depend on B and C assert len(entity_a.incoming_relations) == 1 # C depends on A - - + assert len(entity_b.outgoing_relations) == 1 # Should depend on C assert len(entity_b.incoming_relations) == 1 # A depends on B - - + assert len(entity_c.outgoing_relations) == 1 # Should depend on A - assert len(entity_c.incoming_relations) == 2 # A and B depend on C \ No newline at end of file + assert len(entity_c.incoming_relations) == 2 # A and B depend on C diff --git a/tests/services/sync/test_sync_service.py b/tests/services/sync/test_sync_service.py index 620bfbe8..d167aa9d 100644 --- a/tests/services/sync/test_sync_service.py +++ b/tests/services/sync/test_sync_service.py @@ -2,12 +2,13 @@ import asyncio from pathlib import Path + import pytest from basic_memory.config import ProjectConfig -from basic_memory.services import EntityService -from basic_memory.services.sync.sync_service import SyncService from basic_memory.models import Entity +from basic_memory.services import EntityService +from basic_memory.sync.sync_service import SyncService async def create_test_file(path: Path, content: str = "test content") -> None: