mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix tests
This commit is contained in:
phernandez
@@ -1,162 +0,0 @@
|
||||
"""Service for detecting changes between filesystem and database."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, Sequence
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.file_utils import compute_checksum
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.sync.utils import SyncReport
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileState:
|
||||
"""State of a file including file path, permalink and checksum info."""
|
||||
|
||||
file_path: str
|
||||
permalink: str
|
||||
checksum: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
"""Result of scanning a directory."""
|
||||
|
||||
# file_path -> checksum
|
||||
files: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# checksum -> file_path
|
||||
checksums: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# file_path -> error message
|
||||
errors: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class FileChangeScanner:
|
||||
"""
|
||||
Service for detecting changes between filesystem and database.
|
||||
The filesystem is treated as the source of truth.
|
||||
"""
|
||||
|
||||
def __init__(self, entity_repository: EntityRepository):
|
||||
self.entity_repository = entity_repository
|
||||
|
||||
async def scan_directory(self, directory: Path) -> ScanResult:
|
||||
"""
|
||||
Scan directory for markdown files and their checksums.
|
||||
Only processes .md files, logs and skips others.
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
ScanResult containing found files and any errors
|
||||
"""
|
||||
logger.debug(f"Scanning directory: {directory}")
|
||||
result = ScanResult()
|
||||
|
||||
if not directory.exists():
|
||||
logger.debug(f"Directory does not exist: {directory}")
|
||||
return result
|
||||
|
||||
for path in directory.rglob("*"):
|
||||
if not path.is_file() or not path.name.endswith(".md"):
|
||||
if path.is_file():
|
||||
logger.debug(f"Skipping non-markdown file: {path}")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Get relative path first - used in error reporting if needed
|
||||
rel_path = str(path.relative_to(directory))
|
||||
content = path.read_text()
|
||||
checksum = await compute_checksum(content)
|
||||
result.files[rel_path] = checksum
|
||||
|
||||
except Exception as e:
|
||||
rel_path = str(path.relative_to(directory))
|
||||
result.errors[rel_path] = str(e)
|
||||
logger.error(f"Failed to read {rel_path}: {e}")
|
||||
|
||||
logger.debug(f"Found {len(result.files)} markdown files")
|
||||
if result.errors:
|
||||
logger.warning(f"Encountered {len(result.errors)} errors while scanning")
|
||||
|
||||
return result
|
||||
|
||||
async def find_changes(
|
||||
self, directory: Path, db_file_state: Dict[str, FileState]
|
||||
) -> SyncReport:
|
||||
"""Find changes between filesystem and database."""
|
||||
# Get current files and checksums
|
||||
scan_result = await self.scan_directory(directory)
|
||||
current_files = scan_result.files
|
||||
|
||||
# Build report
|
||||
report = SyncReport(total=len(current_files))
|
||||
|
||||
# Track potentially moved files by checksum
|
||||
files_by_checksum = {} # checksum -> file_path
|
||||
|
||||
# First find potential new files and record checksums
|
||||
for file_path, checksum in current_files.items():
|
||||
logger.debug(f"{file_path} ({checksum[:8]})")
|
||||
|
||||
if file_path not in db_file_state:
|
||||
# Could be new or could be the destination of a move
|
||||
report.new.add(file_path)
|
||||
files_by_checksum[checksum] = file_path
|
||||
elif checksum != db_file_state[file_path].checksum:
|
||||
report.modified.add(file_path)
|
||||
|
||||
report.checksums[file_path] = checksum
|
||||
|
||||
# Now detect moves and deletions
|
||||
for db_file_path, db_state in db_file_state.items():
|
||||
if db_file_path not in current_files:
|
||||
if db_state.checksum in files_by_checksum:
|
||||
# Found a move - file exists at new path with same checksum
|
||||
new_path = files_by_checksum[db_state.checksum]
|
||||
report.moves[db_file_path] = new_path
|
||||
# Remove from new files since it's a move
|
||||
report.new.remove(new_path)
|
||||
else:
|
||||
# Actually deleted
|
||||
report.deleted.add(db_file_path)
|
||||
|
||||
# Log summary
|
||||
logger.debug(f"Total files: {report.total}")
|
||||
logger.debug(f"Changes found: {report.total_changes}")
|
||||
logger.debug(f" New: {len(report.new)}")
|
||||
logger.debug(f" Modified: {len(report.modified)}")
|
||||
logger.debug(f" Moved: {len(report.moves)}")
|
||||
logger.debug(f" Deleted: {len(report.deleted)}")
|
||||
|
||||
if scan_result.errors: # pragma: no cover
|
||||
logger.warning("Files skipped due to errors:")
|
||||
for file_path, error in scan_result.errors.items():
|
||||
logger.warning(f" {file_path}: {error}")
|
||||
|
||||
return report
|
||||
|
||||
async def get_db_file_state(self, db_records: Sequence[Entity]) -> Dict[str, FileState]:
|
||||
"""Get file_path and checksums from database.
|
||||
Args:
|
||||
db_records: database records
|
||||
Returns:
|
||||
Dict mapping file paths to FileState
|
||||
:param db_records: the data from the db
|
||||
"""
|
||||
return {
|
||||
r.file_path: FileState(
|
||||
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum or ""
|
||||
)
|
||||
for r in db_records
|
||||
}
|
||||
|
||||
async def find_knowledge_changes(self, directory: Path) -> SyncReport:
|
||||
"""Find changes in knowledge directory."""
|
||||
db_file_state = await self.get_db_file_state(await self.entity_repository.find_all())
|
||||
return await self.find_changes(directory=directory, db_file_state=db_file_state)
|
||||
@@ -1,174 +0,0 @@
|
||||
"""Service for syncing files between filesystem and database."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory import file_utils
|
||||
from basic_memory.markdown import EntityParser, EntityMarkdown
|
||||
from basic_memory.repository import EntityRepository, RelationRepository
|
||||
from basic_memory.services import EntityService
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync import FileChangeScanner
|
||||
from basic_memory.sync.utils import SyncReport
|
||||
|
||||
|
||||
class SyncService:
|
||||
"""Syncs documents and knowledge files with database.
|
||||
|
||||
Implements two-pass sync strategy for knowledge files to handle relations:
|
||||
1. First pass creates/updates entities without relations
|
||||
2. Second pass processes relations after all entities exist
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scanner: FileChangeScanner,
|
||||
entity_service: EntityService,
|
||||
entity_parser: EntityParser,
|
||||
entity_repository: EntityRepository,
|
||||
relation_repository: RelationRepository,
|
||||
search_service: SearchService,
|
||||
):
|
||||
self.scanner = scanner
|
||||
self.entity_service = entity_service
|
||||
self.entity_parser = entity_parser
|
||||
self.entity_repository = entity_repository
|
||||
self.relation_repository = relation_repository
|
||||
self.search_service = search_service
|
||||
|
||||
async def handle_entity_deletion(self, file_path: str):
|
||||
"""Handle complete entity deletion including search index cleanup."""
|
||||
# First get entity to get permalink before deletion
|
||||
entity = await self.entity_repository.get_by_file_path(file_path)
|
||||
if entity:
|
||||
logger.debug(f"Deleting entity and cleaning up search index: {file_path}")
|
||||
|
||||
# Delete from db (this cascades to observations/relations)
|
||||
await self.entity_service.delete_entity_by_file_path(file_path)
|
||||
|
||||
# Clean up search index
|
||||
permalinks = (
|
||||
[entity.permalink]
|
||||
+ [o.permalink for o in entity.observations]
|
||||
+ [r.permalink for r in entity.relations]
|
||||
)
|
||||
logger.debug(f"Deleting from search index: {permalinks}")
|
||||
for permalink in permalinks:
|
||||
await self.search_service.delete_by_permalink(permalink)
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""Sync knowledge files with database."""
|
||||
|
||||
with logfire.span("sync", directory=directory): # pyright: ignore [reportGeneralTypeIssues]
|
||||
changes = await self.scanner.find_knowledge_changes(directory)
|
||||
logger.info(f"Found {changes.total_changes} knowledge changes")
|
||||
|
||||
# Handle moves first
|
||||
for old_path, new_path in changes.moves.items():
|
||||
logger.debug(f"Moving entity: {old_path} -> {new_path}")
|
||||
entity = await self.entity_repository.get_by_file_path(old_path)
|
||||
if entity:
|
||||
# Update file_path but keep the same permalink for link stability
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
|
||||
)
|
||||
# update search index
|
||||
if updated:
|
||||
await self.search_service.index_entity(updated)
|
||||
|
||||
# Handle deletions next
|
||||
# remove rows from db for files no longer present
|
||||
for path in changes.deleted:
|
||||
await self.handle_entity_deletion(path)
|
||||
|
||||
# Parse files that need updating
|
||||
parsed_entities: Dict[str, EntityMarkdown] = {}
|
||||
|
||||
for path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / path)
|
||||
parsed_entities[path] = entity_markdown
|
||||
|
||||
# First pass: Create/update entities
|
||||
# entities will have a null checksum to indicate they are not complete
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
# Get unique permalink and update markdown if needed
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
Path(path), markdown=entity_markdown
|
||||
)
|
||||
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
# Add/update permalink in frontmatter
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {path}")
|
||||
|
||||
# update markdown
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
|
||||
# update file frontmatter
|
||||
updated_checksum = await file_utils.update_frontmatter(
|
||||
directory / path, {"permalink": permalink}
|
||||
)
|
||||
|
||||
# Update checksum in changes report since file was modified
|
||||
changes.checksums[path] = updated_checksum
|
||||
|
||||
# if the file is new, create an entity
|
||||
if path in changes.new:
|
||||
# Create entity with final permalink
|
||||
logger.debug(f"Creating new entity_markdown: {path}")
|
||||
await self.entity_service.create_entity_from_markdown(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
# otherwise we need to update the entity and observations
|
||||
else:
|
||||
logger.debug(f"Updating entity_markdown: {path}")
|
||||
await self.entity_service.update_entity_and_observations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# Second pass
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {path}")
|
||||
|
||||
# Process relations
|
||||
checksum = changes.checksums[path]
|
||||
entity = await self.entity_service.update_entity_relations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# add to search index
|
||||
await self.search_service.index_entity(entity)
|
||||
|
||||
# Set final checksum to mark sync complete
|
||||
await self.entity_repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
# Third pass: Try to resolve any forward references
|
||||
logger.debug("Attempting to resolve forward references")
|
||||
for relation in await self.relation_repository.find_unresolved_relations():
|
||||
target_entity = await self.entity_service.link_resolver.resolve_link(
|
||||
relation.to_name
|
||||
)
|
||||
# check we found a link that is not the source
|
||||
if target_entity and target_entity.id != relation.from_id:
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
|
||||
)
|
||||
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title, # Update to actual title
|
||||
},
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
|
||||
# update search index
|
||||
await self.search_service.index_entity(target_entity)
|
||||
|
||||
return changes
|
||||
Reference in New Issue
Block a user