admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Beta work (#17)

Browse Source 
feat: Add multiple projects support 
feat: enhanced read_note for when initial result is not found
fix: merge frontmatter when updating note
fix: handle directory removed on sync watch
This commit is contained in:
Paul Hernandez
2025-03-05 18:46:04 -06:00
committed by GitHub
parent 41868fd34c
commit e6496df595
60 changed files with 3506 additions and 1223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/sync/sync_service.py
+270 -32
View File
@@ -1,12 +1,15 @@
"""Service for syncing files between filesystem and database."""
# Suppress logfire warnings
import os
os.environ["LOGFIRE_IGNORE_NO_CONFIG"] = "1"
from dataclasses import dataclass
from dataclasses import field
from datetime import datetime
from pathlib import Path
from typing import Set, Dict
from typing import Tuple
from typing import Dict, Optional, Set, Tuple
import logfire
from loguru import logger
@@ -78,22 +81,126 @@ class SyncService:
self.search_service = search_service
self.file_service = file_service
async def sync(self, directory: Path) -> SyncReport:
@logfire.instrument(extract_args=False)
async def sync(self, directory: Path, show_progress: bool = True) -> SyncReport:
"""Sync all files with database."""
import time
from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn
with logfire.span(f"sync {directory}", directory=directory): # pyright: ignore [reportGeneralTypeIssues]
# initial paths from db to sync
# path -> checksum
report = await self.scan(directory)
start_time = time.time()
console = None
progress = None # Will be initialized if show_progress is True
# order of sync matters to resolve relations effectively
logger.info("Sync operation started", directory=str(directory))
# initial paths from db to sync
# path -> checksum
if show_progress:
from rich.console import Console
console = Console()
console.print(f"Scanning directory: {directory}")
report = await self.scan(directory)
# Initialize progress tracking if requested
if show_progress and report.total > 0:
progress = Progress(
TextColumn("[bold blue]{task.description}"),
BarColumn(),
TaskProgressColumn(),
console=console,
expand=True,
)
# order of sync matters to resolve relations effectively
logger.info(
"Sync changes detected",
new_files=len(report.new),
modified_files=len(report.modified),
deleted_files=len(report.deleted),
moved_files=len(report.moves),
)
if show_progress and report.total > 0:
with progress: # pyright: ignore
# Track each category separately
move_task = None
if report.moves: # pragma: no cover
move_task = progress.add_task("[blue]Moving files...", total=len(report.moves)) # pyright: ignore
delete_task = None
if report.deleted: # pragma: no cover
delete_task = progress.add_task( # pyright: ignore
"[red]Deleting files...", total=len(report.deleted)
)
new_task = None
if report.new:
new_task = progress.add_task( # pyright: ignore
"[green]Adding new files...", total=len(report.new)
)
modify_task = None
if report.modified: # pragma: no cover
modify_task = progress.add_task( # pyright: ignore
"[yellow]Updating modified files...", total=len(report.modified)
)
# sync moves first
for i, (old_path, new_path) in enumerate(report.moves.items()):
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified: # pragma: no cover
report.modified.remove(new_path)
logger.debug(
"File marked as moved and modified",
old_path=old_path,
new_path=new_path,
action="processing as modified",
)
else: # pragma: no cover
await self.handle_move(old_path, new_path)
if move_task is not None: # pragma: no cover
progress.update(move_task, advance=1) # pyright: ignore
# deleted next
for i, path in enumerate(report.deleted): # pragma: no cover
await self.handle_delete(path)
if delete_task is not None: # pragma: no cover
progress.update(delete_task, advance=1) # pyright: ignore
# then new and modified
for i, path in enumerate(report.new):
await self.sync_file(path, new=True)
if new_task is not None:
progress.update(new_task, advance=1) # pyright: ignore
for i, path in enumerate(report.modified): # pragma: no cover
await self.sync_file(path, new=False)
if modify_task is not None: # pragma: no cover
progress.update(modify_task, advance=1) # pyright: ignore
# Final step - resolving relations
if report.total > 0:
relation_task = progress.add_task("[cyan]Resolving relations...", total=1) # pyright: ignore
await self.resolve_relations()
progress.update(relation_task, advance=1) # pyright: ignore
else:
# No progress display - proceed with normal sync
# sync moves first
for old_path, new_path in report.moves.items():
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified:
report.modified.remove(new_path)
logger.debug(
"File marked as moved and modified",
old_path=old_path,
new_path=new_path,
action="processing as modified",
)
else:
await self.handle_move(old_path, new_path)
@@ -109,7 +216,16 @@ class SyncService:
await self.sync_file(path, new=False)
await self.resolve_relations()
return report
duration_ms = int((time.time() - start_time) * 1000)
logger.info(
"Sync operation completed",
directory=str(directory),
total_changes=report.total,
duration_ms=duration_ms,
)
return report
async def scan(self, directory):
"""Scan directory for changes compared to database state."""
@@ -167,25 +283,55 @@ class SyncService:
db_records = await self.entity_repository.find_all()
return {r.file_path: r.checksum or "" for r in db_records}
async def sync_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
"""Sync a single file."""
async def sync_file(
self, path: str, new: bool = True
) -> Tuple[Optional[Entity], Optional[str]]:
"""Sync a single file.
Args:
path: Path to file to sync
new: Whether this is a new file
Returns:
Tuple of (entity, checksum) or (None, None) if sync fails
"""
try:
logger.debug(
"Syncing file",
path=path,
is_new=new,
is_markdown=self.file_service.is_markdown(path),
)
if self.file_service.is_markdown(path):
entity, checksum = await self.sync_markdown_file(path, new)
else:
entity, checksum = await self.sync_regular_file(path, new)
await self.search_service.index_entity(entity)
if entity is not None:
await self.search_service.index_entity(entity)
logger.debug(
"File sync completed", path=path, entity_id=entity.id, checksum=checksum
)
return entity, checksum
except Exception as e: # pragma: no cover
logger.exception(f"Failed to sync {path}: {e}")
return None, None # pyright: ignore
logger.exception("Failed to sync file", path=path, error=str(e))
return None, None
async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
"""Sync a markdown file with full proces sing."""
async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a markdown file with full processing.
Args:
path: Path to markdown file
new: Whether this is a new file
Returns:
Tuple of (entity, checksum)
"""
# Parse markdown first to get any existing permalink
logger.debug("Parsing markdown file", path=path)
entity_markdown = await self.entity_parser.parse_file(path)
# Resolve permalink - this handles all the cases including conflicts
@@ -193,7 +339,13 @@ class SyncService:
# If permalink changed, update the file
if permalink != entity_markdown.frontmatter.permalink:
logger.info(f"Updating permalink in {path}: {permalink}")
logger.info(
"Updating permalink",
path=path,
old_permalink=entity_markdown.frontmatter.permalink,
new_permalink=permalink,
)
entity_markdown.frontmatter.metadata["permalink"] = permalink
checksum = await self.file_service.update_frontmatter(path, {"permalink": permalink})
else:
@@ -202,12 +354,14 @@ class SyncService:
# if the file is new, create an entity
if new:
# Create entity with final permalink
logger.debug(f"Creating new entity from markdown: {path}")
logger.debug("Creating new entity from markdown", path=path, permalink=permalink)
await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
# otherwise we need to update the entity and observations
else:
logger.debug(f"Updating entity from markdown: {path}")
logger.debug("Updating entity from markdown", path=path, permalink=permalink)
await self.entity_service.update_entity_and_observations(Path(path), entity_markdown)
# Update relations and search index
@@ -215,11 +369,27 @@ class SyncService:
# set checksum
await self.entity_repository.update(entity.id, {"checksum": checksum})
logger.debug(
"Markdown sync completed",
path=path,
entity_id=entity.id,
observation_count=len(entity.observations),
relation_count=len(entity.relations),
)
return entity, checksum
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
"""Sync a non-markdown file with basic tracking."""
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a non-markdown file with basic tracking.
Args:
path: Path to file
new: Whether this is a new file
Returns:
Tuple of (entity, checksum)
"""
checksum = await self.file_service.compute_checksum(path)
if new:
# Generate permalink from path
@@ -248,11 +418,18 @@ class SyncService:
return entity, checksum
else:
entity = await self.entity_repository.get_by_file_path(path)
assert entity is not None, "entity should not be None for existing file"
if entity is None: # pragma: no cover
logger.error("Entity not found for existing file", path=path)
raise ValueError(f"Entity not found for existing file: {path}")
updated = await self.entity_repository.update(
entity.id, {"file_path": path, "checksum": checksum}
)
assert updated is not None, "entity should be updated"
if updated is None: # pragma: no cover
logger.error("Failed to update entity", entity_id=entity.id, path=path)
raise ValueError(f"Failed to update entity with ID {entity.id}")
return updated, checksum
async def handle_delete(self, file_path: str):
@@ -261,7 +438,12 @@ class SyncService:
# First get entity to get permalink before deletion
entity = await self.entity_repository.get_by_file_path(file_path)
if entity:
logger.debug(f"Deleting entity and cleaning up search index: {file_path}")
logger.info(
"Deleting entity",
file_path=file_path,
entity_id=entity.id,
permalink=entity.permalink,
)
# Delete from db (this cascades to observations/relations)
await self.entity_service.delete_entity_by_file_path(file_path)
@@ -272,7 +454,14 @@ class SyncService:
+ [o.permalink for o in entity.observations]
+ [r.permalink for r in entity.relations]
)
logger.debug(f"Deleting from search index: {permalinks}")
logger.debug(
"Cleaning up search index",
entity_id=entity.id,
file_path=file_path,
index_entries=len(permalinks),
)
for permalink in permalinks:
if permalink:
await self.search_service.delete_by_permalink(permalink)
@@ -280,12 +469,30 @@ class SyncService:
await self.search_service.delete_by_entity_id(entity.id)
async def handle_move(self, old_path, new_path):
logger.debug(f"Moving entity: {old_path} -> {new_path}")
logger.info("Moving entity", old_path=old_path, new_path=new_path)
entity = await self.entity_repository.get_by_file_path(old_path)
if entity:
# Update file_path but keep the same permalink for link stability
updated = await self.entity_repository.update(entity.id, {"file_path": new_path})
assert updated is not None, "entity should be updated"
if updated is None: # pragma: no cover
logger.error(
"Failed to update entity path",
entity_id=entity.id,
old_path=old_path,
new_path=new_path,
)
raise ValueError(f"Failed to update entity path for ID {entity.id}")
logger.debug(
"Entity path updated",
entity_id=entity.id,
permalink=entity.permalink,
old_path=old_path,
new_path=new_path,
)
# update search index
await self.search_service.index_entity(updated)
@@ -293,14 +500,28 @@ class SyncService:
"""Try to resolve any unresolved relations"""
unresolved_relations = await self.relation_repository.find_unresolved_relations()
logger.debug(f"Attempting to resolve {len(unresolved_relations)} forward references")
logger.info("Resolving forward references", count=len(unresolved_relations))
for relation in unresolved_relations:
logger.debug(
"Attempting to resolve relation",
relation_id=relation.id,
from_id=relation.from_id,
to_name=relation.to_name,
)
resolved_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
# ignore reference to self
if resolved_entity and resolved_entity.id != relation.from_id:
logger.debug(
f"Resolved forward reference: {relation.to_name} -> {resolved_entity.title}"
"Resolved forward reference",
relation_id=relation.id,
from_id=relation.from_id,
to_name=relation.to_name,
resolved_id=resolved_entity.id,
resolved_title=resolved_entity.title,
)
try:
await self.relation_repository.update(
@@ -311,7 +532,12 @@ class SyncService:
},
)
except IntegrityError: # pragma: no cover
logger.debug(f"Ignoring duplicate relation {relation}")
logger.debug(
"Ignoring duplicate relation",
relation_id=relation.id,
from_id=relation.from_id,
to_name=relation.to_name,
)
# update search index
await self.search_service.index_entity(resolved_entity)
@@ -326,8 +552,11 @@ class SyncService:
Returns:
ScanResult containing found files and any errors
"""
import time
logger.debug(f"Scanning directory: {directory}")
start_time = time.time()
logger.debug("Scanning directory", directory=str(directory))
result = ScanResult()
for root, dirnames, filenames in os.walk(str(directory)):
@@ -344,6 +573,15 @@ class SyncService:
checksum = await self.file_service.compute_checksum(rel_path)
result.files[rel_path] = checksum
result.checksums[checksum] = rel_path
logger.debug(f"Found file: {rel_path} with checksum: {checksum}")
logger.debug("Found file", path=rel_path, checksum=checksum)
duration_ms = int((time.time() - start_time) * 1000)
logger.debug(
"Directory scan completed",
directory=str(directory),
files_found=len(result.files),
duration_ms=duration_ms,
)
return result
src/basic_memory/sync/watch_service.py
+151 -28
View File
@@ -1,6 +1,5 @@
"""Watch service for Basic Memory."""
import dataclasses
import os
from datetime import datetime
from pathlib import Path
@@ -29,8 +28,8 @@ class WatchEvent(BaseModel):
class WatchServiceState(BaseModel):
# Service status
running: bool = False
start_time: datetime = dataclasses.field(default_factory=datetime.now)
pid: int = dataclasses.field(default_factory=os.getpid)
start_time: datetime = datetime.now() # Use directly with Pydantic model
pid: int = os.getpid() # Use directly with Pydantic model
# Stats
error_count: int = 0
@@ -41,7 +40,7 @@ class WatchServiceState(BaseModel):
synced_files: int = 0
# Recent activity
recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
recent_events: List[WatchEvent] = [] # Use directly with Pydantic model
def add_event(
self,
@@ -81,10 +80,17 @@ class WatchService:
async def run(self): # pragma: no cover
"""Watch for file changes and sync them"""
logger.info("Watching for sync changes")
logger.info(
"Watch service started",
directory=str(self.config.home),
debounce_ms=self.config.sync_delay,
pid=os.getpid(),
)
self.state.running = True
self.state.start_time = datetime.now()
await self.write_status()
try:
async for changes in awatch(
self.config.home,
@@ -95,14 +101,23 @@ class WatchService:
await self.handle_changes(self.config.home, changes)
except Exception as e:
logger.exception("Watch service error", error=str(e), directory=str(self.config.home))
self.state.record_error(str(e))
await self.write_status()
raise
finally:
logger.info(
"Watch service stopped",
directory=str(self.config.home),
runtime_seconds=int((datetime.now() - self.state.start_time).total_seconds()),
)
self.state.running = False
await self.write_status()
def filter_changes(self, change: Change, path: str) -> bool:
def filter_changes(self, change: Change, path: str) -> bool: # pragma: no cover
"""Filter to only watch non-hidden files and directories.
Returns:
@@ -112,6 +127,7 @@ class WatchService:
try:
relative_path = Path(path).relative_to(self.config.home)
except ValueError:
# This is a defensive check for paths outside our home directory
return False
# Skip hidden directories and files
@@ -128,12 +144,17 @@ class WatchService:
async def handle_changes(self, directory: Path, changes: Set[FileChange]):
"""Process a batch of file changes"""
logger.debug(f"handling {len(changes)} changes in directory: {directory} ...")
import time
from typing import List, Set
start_time = time.time()
logger.info("Processing file changes", change_count=len(changes), directory=str(directory))
# Group changes by type
adds = []
deletes = []
modifies = []
adds: List[str] = []
deletes: List[str] = []
modifies: List[str] = []
for change, path in changes:
# convert to relative path
@@ -145,25 +166,44 @@ class WatchService:
elif change == Change.modified:
modifies.append(relative_path)
logger.debug(
"Grouped file changes", added=len(adds), deleted=len(deletes), modified=len(modifies)
)
# Track processed files to avoid duplicates
processed = set()
processed: Set[str] = set()
# First handle potential moves
for added_path in adds:
if added_path in processed:
continue # pragma: no cover
# Skip directories for added paths
# We don't need to process directories, only the files inside them
# This prevents errors when trying to compute checksums or read directories as files
added_full_path = directory / added_path
if added_full_path.is_dir():
logger.debug("Skipping directory for move detection", path=added_path)
processed.add(added_path)
continue
for deleted_path in deletes:
if deleted_path in processed:
continue # pragma: no cover
# Skip directories for deleted paths (based on entity type in db)
deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
deleted_path
)
if deleted_entity is None:
# If this was a directory, it wouldn't have an entity
logger.debug("Skipping unknown path for move detection", path=deleted_path)
continue
if added_path != deleted_path:
# Compare checksums to detect moves
try:
added_checksum = await self.file_service.compute_checksum(added_path)
deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
deleted_path
)
if deleted_entity and deleted_entity.checksum == added_checksum:
await self.sync_service.handle_move(deleted_path, added_path)
@@ -172,48 +212,131 @@ class WatchService:
action="moved",
status="success",
)
self.console.print(
f"[blue]→[/blue] Moved: {deleted_path} → {added_path}"
)
self.console.print(f"[blue]→[/blue] {deleted_path} → {added_path}")
processed.add(added_path)
processed.add(deleted_path)
break
except Exception as e: # pragma: no cover
logger.warning(f"Error checking for move: {e}")
logger.warning(
"Error checking for move",
old_path=deleted_path,
new_path=added_path,
error=str(e),
)
# Handle remaining changes
# Handle remaining changes - group them by type for concise output
moved_count = len([p for p in processed if p in deletes or p in adds])
delete_count = 0
add_count = 0
modify_count = 0
# Process deletes
for path in deletes:
if path not in processed:
logger.debug("Processing deleted file", path=path)
await self.sync_service.handle_delete(path)
self.state.add_event(path=path, action="deleted", status="success")
self.console.print(f"[red]✕[/red] Deleted: {path}")
self.console.print(f"[red]✕[/red] {path}")
processed.add(path)
delete_count += 1
# Process adds
for path in adds:
if path not in processed:
_, checksum = await self.sync_service.sync_file(path, new=True)
# Skip directories - only process files
full_path = directory / path
if full_path.is_dir(): # pragma: no cover
logger.debug("Skipping directory", path=path)
processed.add(path)
continue
logger.debug("Processing new file", path=path)
entity, checksum = await self.sync_service.sync_file(path, new=True)
if checksum:
self.state.add_event(
path=path, action="new", status="success", checksum=checksum
)
self.console.print(f"[green]✓[/green] Added: {path}")
self.console.print(f"[green]✓[/green] {path}")
logger.debug(
"Added file processed",
path=path,
entity_id=entity.id if entity else None,
checksum=checksum,
)
processed.add(path)
else:
self.console.print(f"[orange]?[/orange] Error syncing: {path}")
add_count += 1
else: # pragma: no cover
logger.warning("Error syncing new file", path=path) # pragma: no cover
self.console.print(
f"[orange]?[/orange] Error syncing: {path}"
) # pragma: no cover
# Process modifies - detect repeats
last_modified_path = None
repeat_count = 0
for path in modifies:
if path not in processed:
_, checksum = await self.sync_service.sync_file(path, new=False)
# Skip directories - only process files
full_path = directory / path
if full_path.is_dir():
logger.debug("Skipping directory", path=path)
processed.add(path)
continue
logger.debug("Processing modified file", path=path)
entity, checksum = await self.sync_service.sync_file(path, new=False)
self.state.add_event(
path=path, action="modified", status="success", checksum=checksum
)
self.console.print(f"[yellow]✎[/yellow] Modified: {path}")
# Check if this is a repeat of the last modified file
if path == last_modified_path: # pragma: no cover
repeat_count += 1 # pragma: no cover
# Only show a message for the first repeat
if repeat_count == 1: # pragma: no cover
self.console.print(
f"[yellow]...[/yellow] Repeated changes to {path}"
) # pragma: no cover
else:
# New file being modified
self.console.print(f"[yellow]✎[/yellow] {path}")
last_modified_path = path
repeat_count = 0
modify_count += 1
logger.debug(
"Modified file processed",
path=path,
entity_id=entity.id if entity else None,
checksum=checksum,
)
processed.add(path)
# Add a divider if we processed any files
# Add a concise summary instead of a divider
if processed:
self.console.print("─" * 80, style="dim")
changes = [] # pyright: ignore
if add_count > 0:
changes.append(f"[green]{add_count} added[/green]") # pyright: ignore
if modify_count > 0:
changes.append(f"[yellow]{modify_count} modified[/yellow]") # pyright: ignore
if moved_count > 0:
changes.append(f"[blue]{moved_count} moved[/blue]") # pyright: ignore
if delete_count > 0:
changes.append(f"[red]{delete_count} deleted[/red]") # pyright: ignore
if changes:
self.console.print(f"{', '.join(changes)}", style="dim") # pyright: ignore
duration_ms = int((time.time() - start_time) * 1000)
self.state.last_scan = datetime.now()
self.state.synced_files += len(processed)
logger.info(
"File change processing completed",
processed_files=len(processed),
total_synced_files=self.state.synced_files,
duration_ms=duration_ms,
)
await self.write_status()