admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Incremental sync on watch (#13)

Browse Source 
- incremental sync on watch
- sync non-markdown files in knowledge base
- experimental `read_resource` tool for reading non-markdown files in raw form (pdf, image)
This commit is contained in:
Paul Hernandez
2025-02-24 22:13:15 -06:00
committed by GitHub
parent 6b8cefcd45
commit 37a01b806d
51 changed files with 2085 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/cli/app.py
-2
View File
@@ -4,9 +4,7 @@ import typer
from basic_memory import db
from basic_memory.config import config
from basic_memory.utils import setup_logging
setup_logging(log_file=".basic-memory/basic-memory-cli.log", console=False) # pragma: no cover
asyncio.run(db.run_migrations(config))
src/basic_memory/cli/commands/status.py
+9 -21
View File
@@ -10,29 +10,16 @@ from rich.console import Console
from rich.panel import Panel
from rich.tree import Tree
from basic_memory import db
from basic_memory.cli.app import app
from basic_memory.cli.commands.sync import get_sync_service
from basic_memory.config import config
from basic_memory.db import DatabaseType
from basic_memory.repository import EntityRepository
from basic_memory.sync import FileChangeScanner
from basic_memory.sync.utils import SyncReport
from basic_memory.sync import SyncService
from basic_memory.sync.sync_service import SyncReport
# Create rich console
console = Console()
async def get_file_change_scanner(
db_type=DatabaseType.FILESYSTEM,
) -> FileChangeScanner: # pragma: no cover
"""Get sync service instance."""
_, session_maker = await db.get_or_create_db(db_path=config.database_path, db_type=db_type)
entity_repository = EntityRepository(session_maker)
file_change_scanner = FileChangeScanner(entity_repository)
return file_change_scanner
def add_files_to_tree(
tree: Tree, paths: Set[str], style: str, checksums: Dict[str, str] | None = None
):
@@ -104,7 +91,7 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
"""Display changes using Rich for better visualization."""
tree = Tree(title)
if changes.total_changes == 0:
if changes.total == 0:
tree.add("No changes")
console.print(Panel(tree, expand=False))
return
@@ -135,11 +122,11 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
console.print(Panel(tree, expand=False))
async def run_status(sync_service: FileChangeScanner, verbose: bool = False):
async def run_status(sync_service: SyncService, verbose: bool = False):
"""Check sync status of files vs database."""
# Check knowledge/ directory
knowledge_changes = await sync_service.find_knowledge_changes(config.home)
display_changes("Knowledge Files", knowledge_changes, verbose)
knowledge_changes = await sync_service.scan(config.home)
display_changes("Status", knowledge_changes, verbose)
@app.command()
@@ -149,8 +136,9 @@ def status(
"""Show sync status between files and database."""
with logfire.span("status"): # pyright: ignore [reportGeneralTypeIssues]
try:
sync_service = asyncio.run(get_file_change_scanner())
sync_service = asyncio.run(get_sync_service())
asyncio.run(run_status(sync_service, verbose)) # pragma: no cover
except Exception as e:
logger.exception(f"Error checking status: {e}")
typer.echo(f"Error checking status: {e}", err=True)
raise typer.Exit(code=1) # pragma: no cover
src/basic_memory/cli/commands/sync.py
+11 -15
View File
@@ -25,8 +25,8 @@ from basic_memory.repository.search_repository import SearchRepository
from basic_memory.services import EntityService, FileService
from basic_memory.services.link_resolver import LinkResolver
from basic_memory.services.search_service import SearchService
from basic_memory.sync import SyncService, FileChangeScanner
from basic_memory.sync.utils import SyncReport
from basic_memory.sync import SyncService
from basic_memory.sync.sync_service import SyncReport
from basic_memory.sync.watch_service import WatchService
console = Console()
@@ -58,9 +58,6 @@ async def get_sync_service(): # pragma: no cover
search_service = SearchService(search_repository, entity_repository, file_service)
link_resolver = LinkResolver(entity_repository, search_service)
# Initialize scanner
file_change_scanner = FileChangeScanner(entity_repository)
# Initialize services
entity_service = EntityService(
entity_parser,
@@ -73,12 +70,12 @@ async def get_sync_service(): # pragma: no cover
# Create sync service
sync_service = SyncService(
scanner=file_change_scanner,
entity_service=entity_service,
entity_parser=entity_parser,
entity_repository=entity_repository,
relation_repository=relation_repository,
search_service=search_service,
file_service=file_service,
)
return sync_service
@@ -95,7 +92,7 @@ def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[V
def display_sync_summary(knowledge: SyncReport):
"""Display a one-line summary of sync changes."""
total_changes = knowledge.total_changes
total_changes = knowledge.total
if total_changes == 0:
console.print("[green]Everything up to date[/green]")
return
@@ -121,13 +118,13 @@ def display_sync_summary(knowledge: SyncReport):
def display_detailed_sync_results(knowledge: SyncReport):
"""Display detailed sync results with trees."""
if knowledge.total_changes == 0:
if knowledge.total == 0:
console.print("\n[green]Everything up to date[/green]")
return
console.print("\n[bold]Sync Results[/bold]")
if knowledge.total_changes > 0:
if knowledge.total > 0:
knowledge_tree = Tree("[bold]Knowledge Files[/bold]")
if knowledge.new:
created = knowledge_tree.add("[green]Created[/green]")
@@ -163,8 +160,10 @@ async def run_sync(verbose: bool = False, watch: bool = False, console_status: b
file_service=sync_service.entity_service.file_service,
config=config,
)
await watch_service.handle_changes(config.home)
await watch_service.run(console_status=console_status) # pragma: no cover
# full sync
await sync_service.sync(config.home)
# watch changes
await watch_service.run() # pragma: no cover
else:
# one time sync
knowledge_changes = await sync_service.sync(config.home)
@@ -189,14 +188,11 @@ def sync(
"-w",
help="Start watching for changes after sync.",
),
console_status: bool = typer.Option(
False, "--console-status", "-c", help="Show live console status"
),
) -> None:
"""Sync knowledge files with the database."""
try:
# Run sync
asyncio.run(run_sync(verbose=verbose, watch=watch, console_status=console_status))
asyncio.run(run_sync(verbose=verbose, watch=watch))
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
src/basic_memory/cli/commands/tools.py
+4 -4
View File
@@ -72,7 +72,7 @@ def build_context(
max_related=max_related,
)
)
rprint(context.model_dump())
rprint(context.model_dump_json(indent=2))
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
typer.echo(f"Error during build_context: {e}", err=True)
@@ -105,7 +105,7 @@ def recent_activity(
max_related=max_related,
)
)
rprint(context.model_dump())
rprint(context.model_dump_json(indent=2))
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
typer.echo(f"Error during build_context: {e}", err=True)
@@ -137,7 +137,7 @@ def search(
after_date=after_date,
)
results = asyncio.run(mcp_search(query=search_query, page=page, page_size=page_size))
rprint(results.model_dump())
rprint(results.model_dump_json(indent=2))
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
typer.echo(f"Error during search: {e}", err=True)
@@ -149,7 +149,7 @@ def search(
def get_entity(identifier: str):
try:
entity = asyncio.run(mcp_get_entity(identifier=identifier))
rprint(entity.model_dump())
rprint(entity.model_dump_json(indent=2))
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
typer.echo(f"Error during get_entity: {e}", err=True)
src/basic_memory/cli/main.py
-1
View File
@@ -15,6 +15,5 @@ from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
tools,
)
if __name__ == "__main__": # pragma: no cover
app()