mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix tests
This commit is contained in:
phernandez
@@ -1,162 +0,0 @@
|
||||
"""Service for detecting changes between filesystem and database."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, Sequence
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.file_utils import compute_checksum
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.sync.utils import SyncReport
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileState:
|
||||
"""State of a file including file path, permalink and checksum info."""
|
||||
|
||||
file_path: str
|
||||
permalink: str
|
||||
checksum: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
"""Result of scanning a directory."""
|
||||
|
||||
# file_path -> checksum
|
||||
files: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# checksum -> file_path
|
||||
checksums: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# file_path -> error message
|
||||
errors: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class FileChangeScanner:
|
||||
"""
|
||||
Service for detecting changes between filesystem and database.
|
||||
The filesystem is treated as the source of truth.
|
||||
"""
|
||||
|
||||
def __init__(self, entity_repository: EntityRepository):
|
||||
self.entity_repository = entity_repository
|
||||
|
||||
async def scan_directory(self, directory: Path) -> ScanResult:
|
||||
"""
|
||||
Scan directory for markdown files and their checksums.
|
||||
Only processes .md files, logs and skips others.
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
ScanResult containing found files and any errors
|
||||
"""
|
||||
logger.debug(f"Scanning directory: {directory}")
|
||||
result = ScanResult()
|
||||
|
||||
if not directory.exists():
|
||||
logger.debug(f"Directory does not exist: {directory}")
|
||||
return result
|
||||
|
||||
for path in directory.rglob("*"):
|
||||
if not path.is_file() or not path.name.endswith(".md"):
|
||||
if path.is_file():
|
||||
logger.debug(f"Skipping non-markdown file: {path}")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Get relative path first - used in error reporting if needed
|
||||
rel_path = str(path.relative_to(directory))
|
||||
content = path.read_text()
|
||||
checksum = await compute_checksum(content)
|
||||
result.files[rel_path] = checksum
|
||||
|
||||
except Exception as e:
|
||||
rel_path = str(path.relative_to(directory))
|
||||
result.errors[rel_path] = str(e)
|
||||
logger.error(f"Failed to read {rel_path}: {e}")
|
||||
|
||||
logger.debug(f"Found {len(result.files)} markdown files")
|
||||
if result.errors:
|
||||
logger.warning(f"Encountered {len(result.errors)} errors while scanning")
|
||||
|
||||
return result
|
||||
|
||||
async def find_changes(
|
||||
self, directory: Path, db_file_state: Dict[str, FileState]
|
||||
) -> SyncReport:
|
||||
"""Find changes between filesystem and database."""
|
||||
# Get current files and checksums
|
||||
scan_result = await self.scan_directory(directory)
|
||||
current_files = scan_result.files
|
||||
|
||||
# Build report
|
||||
report = SyncReport(total=len(current_files))
|
||||
|
||||
# Track potentially moved files by checksum
|
||||
files_by_checksum = {} # checksum -> file_path
|
||||
|
||||
# First find potential new files and record checksums
|
||||
for file_path, checksum in current_files.items():
|
||||
logger.debug(f"{file_path} ({checksum[:8]})")
|
||||
|
||||
if file_path not in db_file_state:
|
||||
# Could be new or could be the destination of a move
|
||||
report.new.add(file_path)
|
||||
files_by_checksum[checksum] = file_path
|
||||
elif checksum != db_file_state[file_path].checksum:
|
||||
report.modified.add(file_path)
|
||||
|
||||
report.checksums[file_path] = checksum
|
||||
|
||||
# Now detect moves and deletions
|
||||
for db_file_path, db_state in db_file_state.items():
|
||||
if db_file_path not in current_files:
|
||||
if db_state.checksum in files_by_checksum:
|
||||
# Found a move - file exists at new path with same checksum
|
||||
new_path = files_by_checksum[db_state.checksum]
|
||||
report.moves[db_file_path] = new_path
|
||||
# Remove from new files since it's a move
|
||||
report.new.remove(new_path)
|
||||
else:
|
||||
# Actually deleted
|
||||
report.deleted.add(db_file_path)
|
||||
|
||||
# Log summary
|
||||
logger.debug(f"Total files: {report.total}")
|
||||
logger.debug(f"Changes found: {report.total_changes}")
|
||||
logger.debug(f" New: {len(report.new)}")
|
||||
logger.debug(f" Modified: {len(report.modified)}")
|
||||
logger.debug(f" Moved: {len(report.moves)}")
|
||||
logger.debug(f" Deleted: {len(report.deleted)}")
|
||||
|
||||
if scan_result.errors: # pragma: no cover
|
||||
logger.warning("Files skipped due to errors:")
|
||||
for file_path, error in scan_result.errors.items():
|
||||
logger.warning(f" {file_path}: {error}")
|
||||
|
||||
return report
|
||||
|
||||
async def get_db_file_state(self, db_records: Sequence[Entity]) -> Dict[str, FileState]:
|
||||
"""Get file_path and checksums from database.
|
||||
Args:
|
||||
db_records: database records
|
||||
Returns:
|
||||
Dict mapping file paths to FileState
|
||||
:param db_records: the data from the db
|
||||
"""
|
||||
return {
|
||||
r.file_path: FileState(
|
||||
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum or ""
|
||||
)
|
||||
for r in db_records
|
||||
}
|
||||
|
||||
async def find_knowledge_changes(self, directory: Path) -> SyncReport:
|
||||
"""Find changes in knowledge directory."""
|
||||
db_file_state = await self.get_db_file_state(await self.entity_repository.find_all())
|
||||
return await self.find_changes(directory=directory, db_file_state=db_file_state)
|
||||
@@ -1,174 +0,0 @@
|
||||
"""Service for syncing files between filesystem and database."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory import file_utils
|
||||
from basic_memory.markdown import EntityParser, EntityMarkdown
|
||||
from basic_memory.repository import EntityRepository, RelationRepository
|
||||
from basic_memory.services import EntityService
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync import FileChangeScanner
|
||||
from basic_memory.sync.utils import SyncReport
|
||||
|
||||
|
||||
class SyncService:
|
||||
"""Syncs documents and knowledge files with database.
|
||||
|
||||
Implements two-pass sync strategy for knowledge files to handle relations:
|
||||
1. First pass creates/updates entities without relations
|
||||
2. Second pass processes relations after all entities exist
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scanner: FileChangeScanner,
|
||||
entity_service: EntityService,
|
||||
entity_parser: EntityParser,
|
||||
entity_repository: EntityRepository,
|
||||
relation_repository: RelationRepository,
|
||||
search_service: SearchService,
|
||||
):
|
||||
self.scanner = scanner
|
||||
self.entity_service = entity_service
|
||||
self.entity_parser = entity_parser
|
||||
self.entity_repository = entity_repository
|
||||
self.relation_repository = relation_repository
|
||||
self.search_service = search_service
|
||||
|
||||
async def handle_entity_deletion(self, file_path: str):
|
||||
"""Handle complete entity deletion including search index cleanup."""
|
||||
# First get entity to get permalink before deletion
|
||||
entity = await self.entity_repository.get_by_file_path(file_path)
|
||||
if entity:
|
||||
logger.debug(f"Deleting entity and cleaning up search index: {file_path}")
|
||||
|
||||
# Delete from db (this cascades to observations/relations)
|
||||
await self.entity_service.delete_entity_by_file_path(file_path)
|
||||
|
||||
# Clean up search index
|
||||
permalinks = (
|
||||
[entity.permalink]
|
||||
+ [o.permalink for o in entity.observations]
|
||||
+ [r.permalink for r in entity.relations]
|
||||
)
|
||||
logger.debug(f"Deleting from search index: {permalinks}")
|
||||
for permalink in permalinks:
|
||||
await self.search_service.delete_by_permalink(permalink)
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""Sync knowledge files with database."""
|
||||
|
||||
with logfire.span("sync", directory=directory): # pyright: ignore [reportGeneralTypeIssues]
|
||||
changes = await self.scanner.find_knowledge_changes(directory)
|
||||
logger.info(f"Found {changes.total_changes} knowledge changes")
|
||||
|
||||
# Handle moves first
|
||||
for old_path, new_path in changes.moves.items():
|
||||
logger.debug(f"Moving entity: {old_path} -> {new_path}")
|
||||
entity = await self.entity_repository.get_by_file_path(old_path)
|
||||
if entity:
|
||||
# Update file_path but keep the same permalink for link stability
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
|
||||
)
|
||||
# update search index
|
||||
if updated:
|
||||
await self.search_service.index_entity(updated)
|
||||
|
||||
# Handle deletions next
|
||||
# remove rows from db for files no longer present
|
||||
for path in changes.deleted:
|
||||
await self.handle_entity_deletion(path)
|
||||
|
||||
# Parse files that need updating
|
||||
parsed_entities: Dict[str, EntityMarkdown] = {}
|
||||
|
||||
for path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / path)
|
||||
parsed_entities[path] = entity_markdown
|
||||
|
||||
# First pass: Create/update entities
|
||||
# entities will have a null checksum to indicate they are not complete
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
# Get unique permalink and update markdown if needed
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
Path(path), markdown=entity_markdown
|
||||
)
|
||||
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
# Add/update permalink in frontmatter
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {path}")
|
||||
|
||||
# update markdown
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
|
||||
# update file frontmatter
|
||||
updated_checksum = await file_utils.update_frontmatter(
|
||||
directory / path, {"permalink": permalink}
|
||||
)
|
||||
|
||||
# Update checksum in changes report since file was modified
|
||||
changes.checksums[path] = updated_checksum
|
||||
|
||||
# if the file is new, create an entity
|
||||
if path in changes.new:
|
||||
# Create entity with final permalink
|
||||
logger.debug(f"Creating new entity_markdown: {path}")
|
||||
await self.entity_service.create_entity_from_markdown(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
# otherwise we need to update the entity and observations
|
||||
else:
|
||||
logger.debug(f"Updating entity_markdown: {path}")
|
||||
await self.entity_service.update_entity_and_observations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# Second pass
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {path}")
|
||||
|
||||
# Process relations
|
||||
checksum = changes.checksums[path]
|
||||
entity = await self.entity_service.update_entity_relations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# add to search index
|
||||
await self.search_service.index_entity(entity)
|
||||
|
||||
# Set final checksum to mark sync complete
|
||||
await self.entity_repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
# Third pass: Try to resolve any forward references
|
||||
logger.debug("Attempting to resolve forward references")
|
||||
for relation in await self.relation_repository.find_unresolved_relations():
|
||||
target_entity = await self.entity_service.link_resolver.resolve_link(
|
||||
relation.to_name
|
||||
)
|
||||
# check we found a link that is not the source
|
||||
if target_entity and target_entity.id != relation.from_id:
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
|
||||
)
|
||||
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title, # Update to actual title
|
||||
},
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
|
||||
# update search index
|
||||
await self.search_service.index_entity(target_entity)
|
||||
|
||||
return changes
|
||||
@@ -1,245 +0,0 @@
|
||||
"""Test file sync service."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.file_utils import compute_checksum
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.sync import FileChangeScanner
|
||||
from basic_memory.sync.file_change_scanner import FileState
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_dir(tmp_path: Path) -> Path:
|
||||
"""Create temp directory for test files."""
|
||||
return tmp_path
|
||||
|
||||
|
||||
async def create_test_file(path: Path, content: str = "test content") -> None:
|
||||
"""Create a test file with given content."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test scanning empty directory."""
|
||||
result = await file_change_scanner.scan_directory(temp_dir)
|
||||
assert len(result.files) == 0
|
||||
assert len(result.errors) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_with_mixed_files(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test scanning directory with markdown and non-markdown files."""
|
||||
# Create test files
|
||||
await create_test_file(temp_dir / "doc.md", "markdown")
|
||||
await create_test_file(temp_dir / "text.txt", "not markdown")
|
||||
await create_test_file(temp_dir / "notes/deep.md", "nested markdown")
|
||||
|
||||
result = await file_change_scanner.scan_directory(temp_dir)
|
||||
assert len(result.files) == 2
|
||||
assert "doc.md" in result.files
|
||||
assert "notes/deep.md" in result.files
|
||||
assert len(result.errors) == 0
|
||||
|
||||
# Verify FileState objects
|
||||
assert isinstance(result.files["doc.md"], str)
|
||||
# checksum
|
||||
assert result.files["doc.md"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_with_unreadable_file(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test scanning directory with an unreadable file."""
|
||||
# Create a file we'll make unreadable
|
||||
bad_file = temp_dir / "bad.md"
|
||||
await create_test_file(bad_file)
|
||||
bad_file.chmod(0o000) # Remove all permissions
|
||||
|
||||
result = await file_change_scanner.scan_directory(temp_dir)
|
||||
assert len(result.files) == 0
|
||||
assert len(result.errors) == 1
|
||||
assert "bad.md" in result.errors
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detect_new_files(
|
||||
file_change_scanner: FileChangeScanner,
|
||||
temp_dir: Path,
|
||||
):
|
||||
"""Test detection of new files."""
|
||||
# Create new file
|
||||
await create_test_file(temp_dir / "new.md")
|
||||
|
||||
# Empty DB state
|
||||
db_records = await file_change_scanner.get_db_file_state([])
|
||||
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
assert len(changes.new) == 1
|
||||
assert "new.md" in changes.new
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detect_modified_file(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test detection of modified files."""
|
||||
file_path = "test.md"
|
||||
content = "original"
|
||||
await create_test_file(temp_dir / file_path, content)
|
||||
|
||||
# Create DB state with original checksum
|
||||
original_checksum = await compute_checksum(content)
|
||||
db_records = {
|
||||
file_path: FileState(file_path=file_path, permalink="test", checksum=original_checksum)
|
||||
}
|
||||
|
||||
# Modify file
|
||||
await create_test_file(temp_dir / file_path, "modified")
|
||||
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
assert len(changes.modified) == 1
|
||||
assert file_path in changes.modified
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test detection of deleted files."""
|
||||
file_path = "deleted.md"
|
||||
|
||||
# Create DB state with file that doesn't exist
|
||||
db_records = {
|
||||
file_path: FileState(file_path=file_path, permalink="deleted", checksum="any-checksum")
|
||||
}
|
||||
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
assert len(changes.deleted) == 1
|
||||
assert file_path in changes.deleted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_db_state_entities(file_change_scanner: FileChangeScanner):
|
||||
"""Test converting entity records to file states."""
|
||||
entity = Entity(permalink="concept/test", file_path="concept/test.md", checksum="test-checksum")
|
||||
|
||||
db_records = await file_change_scanner.get_db_file_state([entity])
|
||||
|
||||
assert len(db_records) == 1
|
||||
assert "concept/test.md" in db_records
|
||||
assert db_records["concept/test.md"].checksum == "test-checksum"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test handling empty/nonexistent directory."""
|
||||
nonexistent = temp_dir / "nonexistent"
|
||||
|
||||
changes = await file_change_scanner.find_changes(directory=nonexistent, db_file_state={})
|
||||
|
||||
assert changes.total_changes == 0
|
||||
assert not changes.new
|
||||
assert not changes.modified
|
||||
assert not changes.deleted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detect_moved_file(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test detection of file moves."""
|
||||
# Create original file
|
||||
old_path = "original/test.md"
|
||||
new_path = "new/location/test.md"
|
||||
content = "test content"
|
||||
|
||||
await create_test_file(temp_dir / old_path, content)
|
||||
original_checksum = await compute_checksum(content)
|
||||
|
||||
# Set up DB state with original location
|
||||
db_records = {
|
||||
old_path: FileState(file_path=old_path, permalink="test", checksum=original_checksum)
|
||||
}
|
||||
|
||||
# Move file to new location
|
||||
old_file = temp_dir / old_path
|
||||
new_file = temp_dir / new_path
|
||||
new_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_file.rename(new_file)
|
||||
|
||||
# Check changes
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
# Should detect as move
|
||||
assert len(changes.moves) == 1
|
||||
assert changes.moves[old_path] == new_path
|
||||
# Should not be in new or deleted
|
||||
assert old_path not in changes.new
|
||||
assert old_path not in changes.deleted
|
||||
assert new_path not in changes.new
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_with_content_change(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test handling a file that is both moved and modified."""
|
||||
# Create original file
|
||||
old_path = "original/test.md"
|
||||
new_path = "new/location/test.md"
|
||||
content = "original content"
|
||||
|
||||
await create_test_file(temp_dir / old_path, content)
|
||||
original_checksum = await compute_checksum(content)
|
||||
|
||||
# Set up DB state with original location
|
||||
db_records = {
|
||||
old_path: FileState(file_path=old_path, permalink="test", checksum=original_checksum)
|
||||
}
|
||||
|
||||
# Move file and change content
|
||||
old_file = temp_dir / old_path
|
||||
new_file = temp_dir / new_path
|
||||
new_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
await create_test_file(new_file, "modified content")
|
||||
old_file.unlink()
|
||||
|
||||
# Check changes
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
# Should be treated as delete + new, not move
|
||||
assert old_path in changes.deleted
|
||||
assert new_path in changes.new
|
||||
assert len(changes.moves) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_moves(file_change_scanner: FileChangeScanner, temp_dir: Path):
|
||||
"""Test detecting multiple file moves at once."""
|
||||
# Create original files
|
||||
files = {"a/test1.md": "content1", "b/test2.md": "content2"}
|
||||
new_locations = {"a/test1.md": "new/test1.md", "b/test2.md": "new/nested/test2.md"}
|
||||
|
||||
db_records = {}
|
||||
# Create files and DB state
|
||||
for old_path, content in files.items():
|
||||
await create_test_file(temp_dir / old_path, content)
|
||||
checksum = await compute_checksum(content)
|
||||
db_records[old_path] = FileState(
|
||||
file_path=old_path, permalink=old_path.replace(".md", ""), checksum=checksum
|
||||
)
|
||||
|
||||
# Move all files
|
||||
for old_path, new_path in new_locations.items():
|
||||
old_file = temp_dir / old_path
|
||||
new_file = temp_dir / new_path
|
||||
new_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_file.rename(new_file)
|
||||
|
||||
# Check changes
|
||||
changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records)
|
||||
|
||||
# Should detect both moves
|
||||
assert len(changes.moves) == 2
|
||||
assert changes.moves["a/test1.md"] == "new/test1.md"
|
||||
assert changes.moves["b/test2.md"] == "new/nested/test2.md"
|
||||
assert not changes.new
|
||||
assert not changes.deleted
|
||||
Reference in New Issue
Block a user