admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor file scanner and tests

Browse Source
This commit is contained in:
phernandez
2025-01-02 15:32:50 -06:00
parent 9c7ccb7117
commit 3baaa323c7
4 changed files with 330 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/services/sync/file_change_scanner.py
+86 -87
View File
@@ -1,13 +1,14 @@
"""Service for detecting changes between filesystem and database."""
from pathlib import Path
from typing import Dict, Protocol, TypeVar, Optional
from typing import Dict, Protocol, TypeVar, Optional, Sequence
from loguru import logger
from basic_memory.models import Document, Entity
from basic_memory.repository.document_repository import DocumentRepository
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.services.sync.utils import FileState, SyncReport
from basic_memory.services.sync.utils import FileState, SyncReport, ScanResult
from basic_memory.utils.file_utils import compute_checksum
@@ -18,7 +19,7 @@ class DbRecord(Protocol):
@property
def path_id(self) -> str: ...
@property
def checksum(self) -> str: ...
def checksum(self) -> Optional[str]: ...
T = TypeVar('T', bound=DbRecord)
@@ -38,7 +39,7 @@ class FileChangeScanner:
self.document_repository = document_repository
self.entity_repository = entity_repository
async def scan_directory(self, directory: Path) -> Dict[str, str]:
async def scan_directory(self, directory: Path) -> ScanResult:
"""
Scan directory for markdown files and their checksums.
Only processes .md files, logs and skips others.
@@ -47,14 +48,14 @@ class FileChangeScanner:
directory: Directory to scan
Returns:
Dict mapping relative paths to checksums
ScanResult containing found files and any errors
"""
logger.debug(f"Scanning directory: {directory}")
files = {}
result = ScanResult()
if not directory.exists():
logger.debug(f"Directory does not exist: {directory}")
return files
return result
for path in directory.rglob("*"):
if not path.is_file() or not path.name.endswith(".md"):
@@ -63,21 +64,36 @@ class FileChangeScanner:
continue
try:
# Get relative path first - used in error reporting if needed
rel_path = str(path.relative_to(directory))
content = path.read_text()
checksum = await compute_checksum(content)
rel_path = str(path.relative_to(directory))
files[rel_path] = checksum
logger.debug(f"Found file: {rel_path} with checksum: {checksum[:8] if checksum else 'None'}")
if checksum: # Only store valid checksums
result.files[rel_path] = FileState(
path=rel_path,
checksum=checksum
)
logger.debug(f"Found file: {rel_path} ({checksum[:8]})")
else:
result.errors[rel_path] = "Failed to compute checksum"
except Exception as e:
logger.error(f"Failed to read {path}: {e}")
rel_path = str(path.relative_to(directory))
result.errors[rel_path] = str(e)
logger.error(f"Failed to read {rel_path}: {e}")
logger.debug(f"Found {len(result.files)} markdown files")
if result.errors:
logger.warning(f"Encountered {len(result.errors)} errors while scanning")
return result
logger.debug(f"Found {len(files)} markdown files")
return files
async def find_changes(
self,
directory: Path,
get_records: callable
db_records: Dict[str, FileState]
) -> SyncReport:
"""
Find changes between filesystem and database.
@@ -90,101 +106,84 @@ class FileChangeScanner:
SyncReport detailing changes
"""
# Get current files and checksums
current_files = await self.scan_directory(directory)
scan_result = await self.scan_directory(directory)
current_files = scan_result.files
logger.debug("Current files from filesystem:")
for path, checksum in sorted(current_files.items()):
logger.debug(f" {path} ({checksum[:8] if checksum else 'No checksum'})")
# Track checksums for display
report = SyncReport()
for path, checksum in current_files.items():
if checksum: # Only store valid checksums
report.checksums[path] = checksum
# Build DB state - use path_id if file_path is NULL
db_records = await get_records()
db_files = {}
for record in db_records:
if record and hasattr(record, 'path_id'): # Guard against None records
path = record.file_path if record.file_path is not None else record.path_id
db_files[path] = (path, record.checksum)
for path, state in sorted(current_files.items()):
logger.debug(f" {path} ({state.checksum[:8]})")
logger.debug("Files from database:")
for path, (_, checksum) in sorted(db_files.items()):
logger.debug(f" {path} ({checksum[:8] if checksum else 'No checksum'})")
for path, state in sorted(db_records.items()):
logger.debug(f" {path} ({state.checksum[:8]})")
# Track files by checksum for move detection
db_paths_by_checksum = {}
for path, (_, checksum) in db_files.items():
if checksum: # Only track valid checksums
paths = db_paths_by_checksum.setdefault(checksum, [])
paths.append(path)
# Build report
report = SyncReport()
# Add current checksums for display
for path, state in current_files.items():
report.checksums[path] = state.checksum
# Find new and modified files
for path, curr_state in current_files.items():
if path not in db_records:
report.new.add(path)
elif curr_state.checksum != db_records[path].checksum:
report.modified.add(path)
processed_current = set()
processed_db = set()
# First pass - check for unchanged and modified files
for curr_path, curr_checksum in current_files.items():
if curr_path in db_files:
_, db_checksum = db_files[curr_path]
processed_current.add(curr_path)
processed_db.add(curr_path)
if curr_checksum != db_checksum:
logger.debug(f"Modified: {curr_path} (checksum changed)")
report.modified.add(curr_path)
# Second pass - look for moves
for curr_path, curr_checksum in current_files.items():
if curr_path in processed_current:
continue
# Look for any files with same checksum in DB
was_move = False
if curr_checksum and curr_checksum in db_paths_by_checksum: # Only check valid checksums
for db_path in db_paths_by_checksum[curr_checksum]:
if db_path not in processed_db:
logger.debug(f"Moved: {db_path} -> {curr_path}")
report.moved[curr_path] = FileState(
path=curr_path,
checksum=curr_checksum,
moved_from=db_path
)
processed_current.add(curr_path)
processed_db.add(db_path)
was_move = True
break
if not was_move:
logger.debug(f"New: {curr_path}")
report.new.add(curr_path)
processed_current.add(curr_path)
# Remaining DB files must be deleted
for path, (db_path, _) in db_files.items():
if path not in processed_db:
logger.debug(f"Deleted: {db_path}")
report.deleted.add(db_path)
# Find deleted files
report.deleted = set(db_records.keys()) - set(current_files.keys())
# Log summary
logger.debug(f"Changes found: {report.total_changes}")
logger.debug(f" New: {len(report.new)}")
logger.debug(f" Modified: {len(report.modified)}")
logger.debug(f" Deleted: {len(report.deleted)}")
logger.debug(f" Moved: {len(report.moved)}")
if scan_result.errors:
logger.warning("Files skipped due to errors:")
for path, error in scan_result.errors.items():
logger.warning(f" {path}: {error}")
return report
async def get_db_state(self, db_records: Sequence[Document | Entity ]) -> Dict[str, FileState]:
"""Get current files and checksums from database.
Args:
get_records: Function to query database records
Returns:
Dict mapping paths to FileState
:param db_records: the data from the db
"""
db_files = {}
for record in db_records:
# Use file_path if available, otherwise use path_id
path = record.file_path if record.file_path is not None else record.path_id
if record.checksum:
db_files[path] = FileState(
path=path,
checksum=record.checksum
)
return db_files
async def find_document_changes(self, directory: Path) -> SyncReport:
"""Find changes in document directory."""
db_records = await self.get_db_state(await self.document_repository.find_all())
return await self.find_changes(
directory=directory,
get_records=self.document_repository.find_all
db_records=db_records
)
async def find_knowledge_changes(self, directory: Path) -> SyncReport:
"""Find changes in knowledge directory."""
db_records = await self.get_db_state(await self.entity_repository.find_all())
return await self.find_changes(
directory=directory,
get_records=self.entity_repository.find_all
db_records=db_records
)
src/basic_memory/services/sync/utils.py
+11 -4
View File
@@ -1,5 +1,7 @@
"""Types and utilities for file sync."""
from dataclasses import dataclass, field
from typing import Optional, Set, Dict
from typing import Set, Dict, Optional
@dataclass
@@ -7,7 +9,13 @@ class FileState:
"""State of a file including path and checksum info."""
path: str
checksum: str
moved_from: Optional[str] = None
@dataclass
class ScanResult:
"""Result of scanning a directory."""
files: Dict[str, FileState] = field(default_factory=dict)
errors: Dict[str, str] = field(default_factory=dict) # path -> error message
@dataclass
@@ -23,8 +31,7 @@ class SyncReport:
new: Set[str] = field(default_factory=set)
modified: Set[str] = field(default_factory=set)
deleted: Set[str] = field(default_factory=set)
moved: Dict[str, FileState] = field(default_factory=dict) # new_path -> state
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
checksums: Dict[str, str] = field(default_factory=dict)
@property
def total_changes(self) -> int: