mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
file_sync_service
This commit is contained in:
phernandez
@@ -12,7 +12,7 @@ from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.repository import DocumentRepository
|
||||
from basic_memory.repository import DocumentRepository, EntityRepository
|
||||
from basic_memory.services import FileSyncService
|
||||
from basic_memory.services.file_sync_service import SyncReport
|
||||
|
||||
@@ -26,7 +26,8 @@ async def get_sync_service(db_type=DatabaseType.FILESYSTEM) -> FileSyncService:
|
||||
session_maker,
|
||||
):
|
||||
document_repository = DocumentRepository(session_maker)
|
||||
sync_service = FileSyncService(document_repository)
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
sync_service = FileSyncService(document_repository, entity_repository)
|
||||
return sync_service
|
||||
|
||||
|
||||
@@ -131,13 +132,11 @@ async def run_status(sync_service: FileSyncService, verbose: bool = False):
|
||||
"""Check sync status of files vs database."""
|
||||
|
||||
# Check knowledge/ directory
|
||||
files = await sync_service.scan_files(config.knowledge_dir)
|
||||
knowledge_changes = await sync_service.find_changes(files, config.knowledge_dir)
|
||||
knowledge_changes = await sync_service.find_knowledge_changes(config.knowledge_dir)
|
||||
display_changes("Knowledge Files", knowledge_changes)
|
||||
|
||||
# Check documents/ directory
|
||||
files = await sync_service.scan_files(config.documents_dir)
|
||||
document_changes = await sync_service.find_changes(files, config.documents_dir)
|
||||
document_changes = await sync_service.find_document_changes(config.documents_dir)
|
||||
display_changes("Documents", document_changes)
|
||||
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
"""Service for syncing files with the database."""
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Set
|
||||
from typing import Set, Dict, Protocol, TypeVar, Generic
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.repository.document_repository import DocumentRepository
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.utils.file_utils import compute_checksum
|
||||
|
||||
|
||||
@@ -22,192 +22,111 @@ class SyncReport:
|
||||
def total_changes(self) -> int:
|
||||
return len(self.new) + len(self.modified) + len(self.deleted)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"Changes detected:\n"
|
||||
f" New files: {len(self.new)}\n"
|
||||
f" Modified: {len(self.modified)}\n"
|
||||
f" Deleted: {len(self.deleted)}"
|
||||
)
|
||||
|
||||
class DbRecord(Protocol):
|
||||
"""Protocol for database records with path and checksum."""
|
||||
@property
|
||||
def path(self) -> str: ...
|
||||
@property
|
||||
def checksum(self) -> str: ...
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
"""Raised when sync operations fail."""
|
||||
pass
|
||||
T = TypeVar('T', bound=DbRecord)
|
||||
|
||||
|
||||
class FileSyncService:
|
||||
"""Service for keeping files and database in sync."""
|
||||
"""
|
||||
Service for keeping files and database in sync.
|
||||
The filesystem is the source of truth.
|
||||
"""
|
||||
|
||||
def __init__(self, document_repository: DocumentRepository):
|
||||
self.repository = document_repository
|
||||
def __init__(
|
||||
self,
|
||||
document_repository: DocumentRepository,
|
||||
entity_repository: EntityRepository
|
||||
):
|
||||
self.document_repository = document_repository
|
||||
self.entity_repository = entity_repository
|
||||
|
||||
async def scan_files(self, directory: Path) -> dict[str, str]:
|
||||
async def scan_directory(self, directory: Path) -> Dict[str, str]:
|
||||
"""
|
||||
Scan directory for markdown files and their checksums.
|
||||
Only processes .md files, logs and skips other files.
|
||||
Only processes .md files, logs and skips others.
|
||||
|
||||
Args:
|
||||
directory: Root directory to scan
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
Dict mapping paths to checksums
|
||||
|
||||
Raises:
|
||||
SyncError: If any markdown file cannot be read
|
||||
Dict mapping relative paths to checksums
|
||||
"""
|
||||
logger.debug(f"Scanning directory: {directory}")
|
||||
files = {}
|
||||
errors = []
|
||||
|
||||
for path in directory.rglob('*'):
|
||||
if path.is_file():
|
||||
if not path.name.endswith('.md'):
|
||||
logger.debug(f"Skipping non-markdown file: '{path}'")
|
||||
continue
|
||||
for path in directory.rglob("*"):
|
||||
if not path.is_file() or not path.name.endswith(".md"):
|
||||
if path.is_file():
|
||||
logger.debug(f"Skipping non-markdown file: {path}")
|
||||
continue
|
||||
|
||||
try:
|
||||
content = path.read_text()
|
||||
checksum = await compute_checksum(content)
|
||||
# Store path relative to root directory
|
||||
rel_path = str(path.relative_to(directory))
|
||||
files[rel_path] = checksum
|
||||
except Exception as e:
|
||||
errors.append(f"Failed to read {path}: {e}")
|
||||
logger.debug(f"Scanned file: {path} checksum: {checksum}")
|
||||
if errors:
|
||||
raise SyncError("Failed to read files:\n" + "\n".join(errors))
|
||||
try:
|
||||
content = path.read_text()
|
||||
checksum = await compute_checksum(content)
|
||||
rel_path = str(path.relative_to(directory))
|
||||
files[rel_path] = checksum
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read {path}: {e}")
|
||||
|
||||
logger.debug(f"Found {len(files)} markdown files in {directory}")
|
||||
logger.debug(f"Found {len(files)} markdown files")
|
||||
return files
|
||||
|
||||
async def find_changes(self, file_system_files: dict[str, str], directory: Path) -> SyncReport:
|
||||
async def find_changes(
|
||||
self,
|
||||
directory: Path,
|
||||
get_records: callable,
|
||||
get_path: callable = lambda x: x.path
|
||||
) -> SyncReport:
|
||||
"""
|
||||
Find changes between filesystem and database.
|
||||
Only considers files that belong to the specified directory.
|
||||
|
||||
Args:
|
||||
file_system_files: Dict mapping paths to checksums
|
||||
directory: Root directory being scanned (knowledge/ or documents/)
|
||||
directory: Directory to check
|
||||
get_records: Function to get database records
|
||||
get_path: Function to get path from record (defaults to .path)
|
||||
|
||||
Returns:
|
||||
SyncReport detailing changes
|
||||
"""
|
||||
logger.debug(f"Finding changes in {directory}")
|
||||
|
||||
# Get all documents from DB
|
||||
db_documents = await self.repository.find_all()
|
||||
|
||||
# Filter DB files to only those in this directory
|
||||
# Get current files and checksums
|
||||
current_files = await self.scan_directory(directory)
|
||||
|
||||
# Get database records
|
||||
db_records = await get_records()
|
||||
db_files = {
|
||||
doc.path: doc.checksum
|
||||
for doc in db_documents
|
||||
if Path(doc.path).is_relative_to(directory.name) # e.g., 'knowledge/' or 'documents/'
|
||||
get_path(record): record.checksum
|
||||
for record in db_records
|
||||
}
|
||||
|
||||
logger.debug(f"Found {len(db_files)} files in DB for {directory}")
|
||||
|
||||
# Find changes
|
||||
new = set(file_system_files.keys()) - set(db_files.keys())
|
||||
deleted = set(db_files.keys()) - set(file_system_files.keys())
|
||||
# Compare current vs database state
|
||||
new = set(current_files.keys()) - set(db_files.keys())
|
||||
deleted = set(db_files.keys()) - set(current_files.keys())
|
||||
modified = {
|
||||
path for path in file_system_files
|
||||
if path in db_files and file_system_files[path] != db_files[path]
|
||||
path for path in current_files
|
||||
if path in db_files and current_files[path] != db_files[path]
|
||||
}
|
||||
|
||||
return SyncReport(new=new, modified=modified, deleted=deleted)
|
||||
|
||||
async def sync_new_file(self, path: str, directory: Path) -> None:
|
||||
"""
|
||||
Sync a new file.
|
||||
async def find_document_changes(self, directory: Path) -> SyncReport:
|
||||
"""Find changes in document directory."""
|
||||
return await self.find_changes(
|
||||
directory=directory,
|
||||
get_records=self.document_repository.find_all
|
||||
)
|
||||
|
||||
Args:
|
||||
path: Relative path to file
|
||||
directory: Root directory
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
full_path = directory / path
|
||||
try:
|
||||
content = full_path.read_text()
|
||||
checksum = await self.compute_checksum(content)
|
||||
await self.repository.create({
|
||||
"path": path,
|
||||
"checksum": checksum
|
||||
})
|
||||
except Exception as e:
|
||||
raise SyncError(f"Failed to sync new file {path}: {e}")
|
||||
|
||||
async def sync_modified_file(self, path: str, directory: Path) -> None:
|
||||
"""
|
||||
Sync a modified file.
|
||||
|
||||
Args:
|
||||
path: Relative path to file
|
||||
directory: Root directory
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
full_path = directory / path
|
||||
try:
|
||||
content = full_path.read_text()
|
||||
checksum = await self.compute_checksum(content)
|
||||
doc = await self.repository.find_by_path(path)
|
||||
if doc:
|
||||
await self.repository.update(doc.id, {"checksum": checksum})
|
||||
else:
|
||||
await self.repository.create({
|
||||
"path": path,
|
||||
"checksum": checksum
|
||||
})
|
||||
except Exception as e:
|
||||
raise SyncError(f"Failed to sync modified file {path}: {e}")
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""
|
||||
Sync filesystem with database.
|
||||
Filesystem is source of truth.
|
||||
|
||||
Args:
|
||||
directory: Root directory to sync
|
||||
|
||||
Returns:
|
||||
SyncReport detailing changes
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
logger.info(f"Starting sync of {directory}")
|
||||
|
||||
# Get current state
|
||||
current_files = await self.scan_files(directory)
|
||||
|
||||
# Find changes
|
||||
changes = await self.find_changes(current_files, directory)
|
||||
logger.info(f"Found changes in {directory}: {changes}")
|
||||
|
||||
if changes.total_changes == 0:
|
||||
logger.info("No changes detected")
|
||||
return changes
|
||||
|
||||
# Process new files
|
||||
for path in changes.new:
|
||||
logger.debug(f"Processing new file: {path}")
|
||||
await self.sync_new_file(path, directory)
|
||||
|
||||
# Process modified files
|
||||
for path in changes.modified:
|
||||
logger.debug(f"Processing modified file: {path}")
|
||||
await self.sync_modified_file(path, directory)
|
||||
|
||||
# Process deleted files
|
||||
for path in changes.deleted:
|
||||
logger.debug(f"Processing deleted file: {path}")
|
||||
doc = await self.repository.find_by_path(path)
|
||||
if doc:
|
||||
await self.repository.delete(doc.id)
|
||||
|
||||
logger.info("Sync completed successfully")
|
||||
return changes
|
||||
async def find_knowledge_changes(self, directory: Path) -> SyncReport:
|
||||
"""Find changes in knowledge directory."""
|
||||
return await self.find_changes(
|
||||
directory=directory,
|
||||
get_records=self.entity_repository.find_all,
|
||||
get_path=lambda x: x.path_id
|
||||
)
|
||||
Reference in New Issue
Block a user