mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
feat: SPEC-20 Simplified Project-Scoped Rclone Sync (#405)
Signed-off-by: phernandez <paul@basicmachines.co> Signed-off-by: Claude <noreply@anthropic.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
@@ -18,6 +18,7 @@ from basic_memory.schemas.project_info import (
|
||||
ProjectInfoRequest,
|
||||
ProjectStatusResponse,
|
||||
)
|
||||
from basic_memory.utils import normalize_project_path
|
||||
|
||||
# Router for resources in a specific project
|
||||
# The ProjectPathDep is used in the path as a prefix, so the request path is like /{project}/project/info
|
||||
@@ -50,7 +51,7 @@ async def get_project(
|
||||
|
||||
return ProjectItem(
|
||||
name=found_project.name,
|
||||
path=found_project.path,
|
||||
path=normalize_project_path(found_project.path),
|
||||
is_default=found_project.is_default or False,
|
||||
)
|
||||
|
||||
@@ -109,6 +110,9 @@ async def sync_project(
|
||||
background_tasks: BackgroundTasks,
|
||||
sync_service: SyncServiceDep,
|
||||
project_config: ProjectConfigDep,
|
||||
force_full: bool = Query(
|
||||
False, description="Force full scan, bypassing watermark optimization"
|
||||
),
|
||||
):
|
||||
"""Force project filesystem sync to database.
|
||||
|
||||
@@ -118,12 +122,17 @@ async def sync_project(
|
||||
background_tasks: FastAPI background tasks
|
||||
sync_service: Sync service for this project
|
||||
project_config: Project configuration
|
||||
force_full: If True, force a full scan even if watermark exists
|
||||
|
||||
Returns:
|
||||
Response confirming sync was initiated
|
||||
"""
|
||||
background_tasks.add_task(sync_service.sync, project_config.home, project_config.name)
|
||||
logger.info(f"Filesystem sync initiated for project: {project_config.name}")
|
||||
background_tasks.add_task(
|
||||
sync_service.sync, project_config.home, project_config.name, force_full=force_full
|
||||
)
|
||||
logger.info(
|
||||
f"Filesystem sync initiated for project: {project_config.name} (force_full={force_full})"
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "sync_started",
|
||||
@@ -167,7 +176,7 @@ async def list_projects(
|
||||
project_items = [
|
||||
ProjectItem(
|
||||
name=project.name,
|
||||
path=project.path,
|
||||
path=normalize_project_path(project.path),
|
||||
is_default=project.is_default or False,
|
||||
)
|
||||
for project in projects
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
"""CLI commands for basic-memory."""
|
||||
|
||||
from . import status, sync, db, import_memory_json, mcp, import_claude_conversations
|
||||
from . import status, db, import_memory_json, mcp, import_claude_conversations
|
||||
from . import import_claude_projects, import_chatgpt, tool, project
|
||||
|
||||
__all__ = [
|
||||
"status",
|
||||
"sync",
|
||||
"db",
|
||||
"import_memory_json",
|
||||
"mcp",
|
||||
|
||||
@@ -1,33 +1,11 @@
|
||||
"""Cloud bisync commands for Basic Memory CLI."""
|
||||
"""Cloud bisync utility functions for Basic Memory CLI."""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.commands.cloud.api_client import CloudAPIError, make_api_request
|
||||
from basic_memory.cli.commands.cloud.cloud_utils import (
|
||||
create_cloud_project,
|
||||
fetch_cloud_projects,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.rclone_config import (
|
||||
add_tenant_to_rclone_config,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import RcloneInstallError, install_rclone
|
||||
from basic_memory.cli.commands.cloud.api_client import make_api_request
|
||||
from basic_memory.config import ConfigManager
|
||||
from basic_memory.ignore_utils import get_bmignore_path, create_default_bmignore
|
||||
from basic_memory.schemas.cloud import (
|
||||
TenantMountInfo,
|
||||
MountCredentials,
|
||||
)
|
||||
|
||||
console = Console()
|
||||
from basic_memory.ignore_utils import create_default_bmignore, get_bmignore_path
|
||||
from basic_memory.schemas.cloud import MountCredentials, TenantMountInfo
|
||||
|
||||
|
||||
class BisyncError(Exception):
|
||||
@@ -36,52 +14,6 @@ class BisyncError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class RcloneBisyncProfile:
|
||||
"""Bisync profile with safety settings."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
conflict_resolve: str,
|
||||
max_delete: int,
|
||||
check_access: bool,
|
||||
description: str,
|
||||
extra_args: Optional[list[str]] = None,
|
||||
):
|
||||
self.name = name
|
||||
self.conflict_resolve = conflict_resolve
|
||||
self.max_delete = max_delete
|
||||
self.check_access = check_access
|
||||
self.description = description
|
||||
self.extra_args = extra_args or []
|
||||
|
||||
|
||||
# Bisync profiles based on SPEC-9 Phase 2.1
|
||||
BISYNC_PROFILES = {
|
||||
"safe": RcloneBisyncProfile(
|
||||
name="safe",
|
||||
conflict_resolve="none",
|
||||
max_delete=10,
|
||||
check_access=False,
|
||||
description="Safe mode with conflict preservation (keeps both versions)",
|
||||
),
|
||||
"balanced": RcloneBisyncProfile(
|
||||
name="balanced",
|
||||
conflict_resolve="newer",
|
||||
max_delete=25,
|
||||
check_access=False,
|
||||
description="Balanced mode - auto-resolve to newer file (recommended)",
|
||||
),
|
||||
"fast": RcloneBisyncProfile(
|
||||
name="fast",
|
||||
conflict_resolve="newer",
|
||||
max_delete=50,
|
||||
check_access=False,
|
||||
description="Fast mode for rapid iteration (skip verification)",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def get_mount_info() -> TenantMountInfo:
|
||||
"""Get current tenant information from cloud API."""
|
||||
try:
|
||||
@@ -110,75 +42,6 @@ async def generate_mount_credentials(tenant_id: str) -> MountCredentials:
|
||||
raise BisyncError(f"Failed to generate credentials: {e}") from e
|
||||
|
||||
|
||||
def scan_local_directories(sync_dir: Path) -> list[str]:
|
||||
"""Scan local sync directory for project folders.
|
||||
|
||||
Args:
|
||||
sync_dir: Path to bisync directory
|
||||
|
||||
Returns:
|
||||
List of directory names (project names)
|
||||
"""
|
||||
if not sync_dir.exists():
|
||||
return []
|
||||
|
||||
directories = []
|
||||
for item in sync_dir.iterdir():
|
||||
if item.is_dir() and not item.name.startswith("."):
|
||||
directories.append(item.name)
|
||||
|
||||
return directories
|
||||
|
||||
|
||||
def get_bisync_state_path(tenant_id: str) -> Path:
|
||||
"""Get path to bisync state directory."""
|
||||
return Path.home() / ".basic-memory" / "bisync-state" / tenant_id
|
||||
|
||||
|
||||
def get_bisync_directory() -> Path:
|
||||
"""Get bisync directory from config.
|
||||
|
||||
Returns:
|
||||
Path to bisync directory (default: ~/basic-memory-cloud-sync)
|
||||
"""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
|
||||
sync_dir = config.bisync_config.get("sync_dir", str(Path.home() / "basic-memory-cloud-sync"))
|
||||
return Path(sync_dir).expanduser().resolve()
|
||||
|
||||
|
||||
def validate_bisync_directory(bisync_dir: Path) -> None:
|
||||
"""Validate bisync directory doesn't conflict with mount.
|
||||
|
||||
Raises:
|
||||
BisyncError: If bisync directory conflicts with mount directory
|
||||
"""
|
||||
# Get fixed mount directory
|
||||
mount_dir = (Path.home() / "basic-memory-cloud").resolve()
|
||||
|
||||
# Check if bisync dir is the same as mount dir
|
||||
if bisync_dir == mount_dir:
|
||||
raise BisyncError(
|
||||
f"Cannot use {bisync_dir} for bisync - it's the mount directory!\n"
|
||||
f"Mount and bisync must use different directories.\n\n"
|
||||
f"Options:\n"
|
||||
f" 1. Use default: ~/basic-memory-cloud-sync/\n"
|
||||
f" 2. Specify different directory: --dir ~/my-sync-folder"
|
||||
)
|
||||
|
||||
# Check if mount is active at this location
|
||||
result = subprocess.run(["mount"], capture_output=True, text=True)
|
||||
if str(bisync_dir) in result.stdout and "rclone" in result.stdout:
|
||||
raise BisyncError(
|
||||
f"{bisync_dir} is currently mounted via 'bm cloud mount'\n"
|
||||
f"Cannot use mounted directory for bisync.\n\n"
|
||||
f"Either:\n"
|
||||
f" 1. Unmount first: bm cloud unmount\n"
|
||||
f" 2. Use different directory for bisync"
|
||||
)
|
||||
|
||||
|
||||
def convert_bmignore_to_rclone_filters() -> Path:
|
||||
"""Convert .bmignore patterns to rclone filter format.
|
||||
|
||||
@@ -245,521 +108,3 @@ def get_bisync_filter_path() -> Path:
|
||||
Path to rclone filter file
|
||||
"""
|
||||
return convert_bmignore_to_rclone_filters()
|
||||
|
||||
|
||||
def bisync_state_exists(tenant_id: str) -> bool:
|
||||
"""Check if bisync state exists (has been initialized)."""
|
||||
state_path = get_bisync_state_path(tenant_id)
|
||||
return state_path.exists() and any(state_path.iterdir())
|
||||
|
||||
|
||||
def build_bisync_command(
|
||||
tenant_id: str,
|
||||
bucket_name: str,
|
||||
local_path: Path,
|
||||
profile: RcloneBisyncProfile,
|
||||
dry_run: bool = False,
|
||||
resync: bool = False,
|
||||
verbose: bool = False,
|
||||
) -> list[str]:
|
||||
"""Build rclone bisync command with profile settings."""
|
||||
|
||||
# Sync with the entire bucket root (all projects)
|
||||
rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}"
|
||||
filter_path = get_bisync_filter_path()
|
||||
state_path = get_bisync_state_path(tenant_id)
|
||||
|
||||
# Ensure state directory exists
|
||||
state_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"bisync",
|
||||
str(local_path),
|
||||
rclone_remote,
|
||||
"--create-empty-src-dirs",
|
||||
"--resilient",
|
||||
f"--conflict-resolve={profile.conflict_resolve}",
|
||||
f"--max-delete={profile.max_delete}",
|
||||
"--filters-file",
|
||||
str(filter_path),
|
||||
"--workdir",
|
||||
str(state_path),
|
||||
]
|
||||
|
||||
# Add verbosity flags
|
||||
if verbose:
|
||||
cmd.append("--verbose") # Full details with file-by-file output
|
||||
else:
|
||||
# Show progress bar during transfers
|
||||
cmd.append("--progress")
|
||||
|
||||
if profile.check_access:
|
||||
cmd.append("--check-access")
|
||||
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
if resync:
|
||||
cmd.append("--resync")
|
||||
|
||||
cmd.extend(profile.extra_args)
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def setup_cloud_bisync(sync_dir: Optional[str] = None) -> None:
|
||||
"""Set up cloud bisync with rclone installation and configuration.
|
||||
|
||||
Args:
|
||||
sync_dir: Optional custom sync directory path. If not provided, uses config default.
|
||||
"""
|
||||
console.print("[bold blue]Basic Memory Cloud Bisync Setup[/bold blue]")
|
||||
console.print("Setting up bidirectional sync to your cloud tenant...\n")
|
||||
|
||||
try:
|
||||
# Step 1: Install rclone
|
||||
console.print("[blue]Step 1: Installing rclone...[/blue]")
|
||||
install_rclone()
|
||||
|
||||
# Step 2: Get mount info (for tenant_id, bucket)
|
||||
console.print("\n[blue]Step 2: Getting tenant information...[/blue]")
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
|
||||
tenant_id = tenant_info.tenant_id
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
console.print(f"[green]✓ Found tenant: {tenant_id}[/green]")
|
||||
console.print(f"[green]✓ Bucket: {bucket_name}[/green]")
|
||||
|
||||
# Step 3: Generate credentials
|
||||
console.print("\n[blue]Step 3: Generating sync credentials...[/blue]")
|
||||
creds = asyncio.run(generate_mount_credentials(tenant_id))
|
||||
|
||||
access_key = creds.access_key
|
||||
secret_key = creds.secret_key
|
||||
|
||||
console.print("[green]✓ Generated secure credentials[/green]")
|
||||
|
||||
# Step 4: Configure rclone
|
||||
console.print("\n[blue]Step 4: Configuring rclone...[/blue]")
|
||||
add_tenant_to_rclone_config(
|
||||
tenant_id=tenant_id,
|
||||
bucket_name=bucket_name,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
)
|
||||
|
||||
# Step 5: Configure and create local directory
|
||||
console.print("\n[blue]Step 5: Configuring sync directory...[/blue]")
|
||||
|
||||
# If custom sync_dir provided, save to config
|
||||
if sync_dir:
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
config.bisync_config["sync_dir"] = sync_dir
|
||||
config_manager.save_config(config)
|
||||
console.print("[green]✓ Saved custom sync directory to config[/green]")
|
||||
|
||||
# Get bisync directory (from config or default)
|
||||
local_path = get_bisync_directory()
|
||||
|
||||
# Validate bisync directory
|
||||
validate_bisync_directory(local_path)
|
||||
|
||||
# Create directory
|
||||
local_path.mkdir(parents=True, exist_ok=True)
|
||||
console.print(f"[green]✓ Created sync directory: {local_path}[/green]")
|
||||
|
||||
# Step 6: Perform initial resync
|
||||
console.print("\n[blue]Step 6: Performing initial sync...[/blue]")
|
||||
console.print("[yellow]This will establish the baseline for bidirectional sync.[/yellow]")
|
||||
|
||||
run_bisync(
|
||||
tenant_id=tenant_id,
|
||||
bucket_name=bucket_name,
|
||||
local_path=local_path,
|
||||
profile_name="balanced",
|
||||
resync=True,
|
||||
)
|
||||
|
||||
console.print("\n[bold green]✓ Bisync setup completed successfully![/bold green]")
|
||||
console.print("\nYour local files will now sync bidirectionally with the cloud!")
|
||||
console.print(f"\nLocal directory: {local_path}")
|
||||
console.print("\nUseful commands:")
|
||||
console.print(" bm sync # Run sync (recommended)")
|
||||
console.print(" bm sync --watch # Start watch mode")
|
||||
console.print(" bm cloud status # Check sync status")
|
||||
console.print(" bm cloud check # Verify file integrity")
|
||||
console.print(" bm cloud bisync --dry-run # Preview changes (advanced)")
|
||||
|
||||
except (RcloneInstallError, BisyncError, CloudAPIError) as e:
|
||||
console.print(f"\n[red]Setup failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def run_bisync(
|
||||
tenant_id: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
local_path: Optional[Path] = None,
|
||||
profile_name: str = "balanced",
|
||||
dry_run: bool = False,
|
||||
resync: bool = False,
|
||||
verbose: bool = False,
|
||||
) -> bool:
|
||||
"""Run rclone bisync with specified profile."""
|
||||
|
||||
try:
|
||||
# Get tenant info if not provided
|
||||
if not tenant_id or not bucket_name:
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
tenant_id = tenant_info.tenant_id
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Set default local path if not provided
|
||||
if not local_path:
|
||||
local_path = get_bisync_directory()
|
||||
|
||||
# Validate bisync directory
|
||||
validate_bisync_directory(local_path)
|
||||
|
||||
# Check if local path exists
|
||||
if not local_path.exists():
|
||||
raise BisyncError(
|
||||
f"Local directory {local_path} does not exist. Run 'basic-memory cloud bisync-setup' first."
|
||||
)
|
||||
|
||||
# Get bisync profile
|
||||
if profile_name not in BISYNC_PROFILES:
|
||||
raise BisyncError(
|
||||
f"Unknown profile: {profile_name}. Available: {list(BISYNC_PROFILES.keys())}"
|
||||
)
|
||||
|
||||
profile = BISYNC_PROFILES[profile_name]
|
||||
|
||||
# Auto-register projects before sync (unless dry-run or resync)
|
||||
if not dry_run and not resync:
|
||||
try:
|
||||
console.print("[dim]Checking for new projects...[/dim]")
|
||||
|
||||
# Fetch cloud projects and extract directory names from paths
|
||||
cloud_data = asyncio.run(fetch_cloud_projects())
|
||||
cloud_projects = cloud_data.projects
|
||||
|
||||
# Extract directory names from cloud project paths
|
||||
# Compare directory names, not project names
|
||||
# Cloud path /app/data/basic-memory -> directory name "basic-memory"
|
||||
cloud_dir_names = set()
|
||||
for p in cloud_projects:
|
||||
path = p.path
|
||||
# Strip /app/data/ prefix if present (cloud mode)
|
||||
if path.startswith("/app/data/"):
|
||||
path = path[len("/app/data/") :]
|
||||
# Get the last segment (directory name)
|
||||
dir_name = Path(path).name
|
||||
cloud_dir_names.add(dir_name)
|
||||
|
||||
# Scan local directories
|
||||
local_dirs = scan_local_directories(local_path)
|
||||
|
||||
# Create missing cloud projects
|
||||
new_projects = []
|
||||
for dir_name in local_dirs:
|
||||
if dir_name not in cloud_dir_names:
|
||||
new_projects.append(dir_name)
|
||||
|
||||
if new_projects:
|
||||
console.print(
|
||||
f"[blue]Found {len(new_projects)} new local project(s), creating on cloud...[/blue]"
|
||||
)
|
||||
for project_name in new_projects:
|
||||
try:
|
||||
asyncio.run(create_cloud_project(project_name))
|
||||
console.print(f"[green] ✓ Created project: {project_name}[/green]")
|
||||
except BisyncError as e:
|
||||
console.print(
|
||||
f"[yellow] ⚠ Could not create {project_name}: {e}[/yellow]"
|
||||
)
|
||||
else:
|
||||
console.print("[dim]All local projects already registered on cloud[/dim]")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Project auto-registration failed: {e}[/yellow]")
|
||||
console.print("[yellow]Continuing with sync anyway...[/yellow]")
|
||||
|
||||
# Check if first run and require resync
|
||||
if not resync and not bisync_state_exists(tenant_id) and not dry_run:
|
||||
raise BisyncError(
|
||||
"First bisync requires --resync to establish baseline. "
|
||||
"Run: basic-memory cloud bisync --resync"
|
||||
)
|
||||
|
||||
# Build and execute bisync command
|
||||
bisync_cmd = build_bisync_command(
|
||||
tenant_id,
|
||||
bucket_name,
|
||||
local_path,
|
||||
profile,
|
||||
dry_run=dry_run,
|
||||
resync=resync,
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
console.print("[yellow]DRY RUN MODE - No changes will be made[/yellow]")
|
||||
|
||||
console.print(
|
||||
f"[blue]Running bisync with profile '{profile_name}' ({profile.description})...[/blue]"
|
||||
)
|
||||
console.print(f"[dim]Command: {' '.join(bisync_cmd)}[/dim]")
|
||||
console.print() # Blank line before output
|
||||
|
||||
# Stream output in real-time so user sees progress
|
||||
result = subprocess.run(bisync_cmd, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise BisyncError(f"Bisync command failed with code {result.returncode}")
|
||||
|
||||
console.print() # Blank line after output
|
||||
|
||||
if dry_run:
|
||||
console.print("[green]✓ Dry run completed successfully[/green]")
|
||||
elif resync:
|
||||
console.print("[green]✓ Initial sync baseline established[/green]")
|
||||
else:
|
||||
console.print("[green]✓ Sync completed successfully[/green]")
|
||||
|
||||
# Notify container to refresh cache (if not dry run)
|
||||
if not dry_run:
|
||||
try:
|
||||
asyncio.run(notify_container_sync(tenant_id))
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Could not notify container: {e}[/yellow]")
|
||||
|
||||
return True
|
||||
|
||||
except BisyncError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise BisyncError(f"Unexpected error during bisync: {e}") from e
|
||||
|
||||
|
||||
async def notify_container_sync(tenant_id: str) -> None:
|
||||
"""Sync all projects after bisync completes."""
|
||||
try:
|
||||
from basic_memory.cli.commands.command_utils import run_sync
|
||||
|
||||
# Fetch all projects and sync each one
|
||||
cloud_data = await fetch_cloud_projects()
|
||||
projects = cloud_data.projects
|
||||
|
||||
if not projects:
|
||||
console.print("[dim]No projects to sync[/dim]")
|
||||
return
|
||||
|
||||
console.print(f"[blue]Notifying cloud to index {len(projects)} project(s)...[/blue]")
|
||||
|
||||
for project in projects:
|
||||
project_name = project.name
|
||||
if project_name:
|
||||
try:
|
||||
await run_sync(project=project_name)
|
||||
except Exception as e:
|
||||
# Non-critical, log and continue
|
||||
console.print(f"[yellow] ⚠ Sync failed for {project_name}: {e}[/yellow]")
|
||||
|
||||
console.print("[dim]Note: Cloud indexing has started and may take a few moments[/dim]")
|
||||
|
||||
except Exception as e:
|
||||
# Non-critical, don't fail the bisync
|
||||
console.print(f"[yellow]Warning: Post-sync failed: {e}[/yellow]")
|
||||
|
||||
|
||||
def run_bisync_watch(
|
||||
tenant_id: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
local_path: Optional[Path] = None,
|
||||
profile_name: str = "balanced",
|
||||
interval_seconds: int = 60,
|
||||
) -> None:
|
||||
"""Run bisync in watch mode with periodic syncs."""
|
||||
|
||||
console.print("[bold blue]Starting bisync watch mode[/bold blue]")
|
||||
console.print(f"Sync interval: {interval_seconds} seconds")
|
||||
console.print("Press Ctrl+C to stop\n")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
run_bisync(
|
||||
tenant_id=tenant_id,
|
||||
bucket_name=bucket_name,
|
||||
local_path=local_path,
|
||||
profile_name=profile_name,
|
||||
)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
console.print(f"[dim]Sync completed in {elapsed:.1f}s[/dim]")
|
||||
|
||||
# Wait for next interval
|
||||
time.sleep(interval_seconds)
|
||||
|
||||
except BisyncError as e:
|
||||
console.print(f"[red]Sync error: {e}[/red]")
|
||||
console.print(f"[yellow]Retrying in {interval_seconds} seconds...[/yellow]")
|
||||
time.sleep(interval_seconds)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[yellow]Watch mode stopped[/yellow]")
|
||||
|
||||
|
||||
def show_bisync_status() -> None:
|
||||
"""Show current bisync status and configuration."""
|
||||
|
||||
try:
|
||||
# Get tenant info
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
tenant_id = tenant_info.tenant_id
|
||||
|
||||
local_path = get_bisync_directory()
|
||||
state_path = get_bisync_state_path(tenant_id)
|
||||
|
||||
# Create status table
|
||||
table = Table(title="Cloud Bisync Status", show_header=True, header_style="bold blue")
|
||||
table.add_column("Property", style="green", min_width=20)
|
||||
table.add_column("Value", style="dim", min_width=30)
|
||||
|
||||
# Check initialization status
|
||||
is_initialized = bisync_state_exists(tenant_id)
|
||||
init_status = (
|
||||
"[green]✓ Initialized[/green]" if is_initialized else "[red]✗ Not initialized[/red]"
|
||||
)
|
||||
|
||||
table.add_row("Tenant ID", tenant_id)
|
||||
table.add_row("Local Directory", str(local_path))
|
||||
table.add_row("Status", init_status)
|
||||
table.add_row("State Directory", str(state_path))
|
||||
|
||||
# Check for last sync info
|
||||
if is_initialized:
|
||||
# Look for most recent state file
|
||||
state_files = list(state_path.glob("*.lst"))
|
||||
if state_files:
|
||||
latest = max(state_files, key=lambda p: p.stat().st_mtime)
|
||||
last_sync = datetime.fromtimestamp(latest.stat().st_mtime)
|
||||
table.add_row("Last Sync", last_sync.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show bisync profiles
|
||||
console.print("\n[bold]Available bisync profiles:[/bold]")
|
||||
for name, profile in BISYNC_PROFILES.items():
|
||||
console.print(f" {name}: {profile.description}")
|
||||
console.print(f" - Conflict resolution: {profile.conflict_resolve}")
|
||||
console.print(f" - Max delete: {profile.max_delete} files")
|
||||
|
||||
console.print("\n[dim]To use a profile: bm cloud bisync --profile <name>[/dim]")
|
||||
|
||||
# Show setup instructions if not initialized
|
||||
if not is_initialized:
|
||||
console.print("\n[yellow]To initialize bisync, run:[/yellow]")
|
||||
console.print(" bm cloud setup")
|
||||
console.print(" or")
|
||||
console.print(" bm cloud bisync --resync")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error getting bisync status: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def run_check(
|
||||
tenant_id: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
local_path: Optional[Path] = None,
|
||||
one_way: bool = False,
|
||||
) -> bool:
|
||||
"""Check file integrity between local and cloud using rclone check.
|
||||
|
||||
Args:
|
||||
tenant_id: Cloud tenant ID (auto-detected if not provided)
|
||||
bucket_name: S3 bucket name (auto-detected if not provided)
|
||||
local_path: Local bisync directory (uses config default if not provided)
|
||||
one_way: If True, only check for missing files on destination (faster)
|
||||
|
||||
Returns:
|
||||
True if check passed (files match), False if differences found
|
||||
"""
|
||||
try:
|
||||
# Check if rclone is installed
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed
|
||||
|
||||
if not is_rclone_installed():
|
||||
raise BisyncError(
|
||||
"rclone is not installed. Run 'bm cloud bisync-setup' first to set up cloud sync."
|
||||
)
|
||||
|
||||
# Get tenant info if not provided
|
||||
if not tenant_id or not bucket_name:
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
tenant_id = tenant_id or tenant_info.tenant_id
|
||||
bucket_name = bucket_name or tenant_info.bucket_name
|
||||
|
||||
# Get local path from config
|
||||
if not local_path:
|
||||
local_path = get_bisync_directory()
|
||||
|
||||
# Check if bisync is initialized
|
||||
if not bisync_state_exists(tenant_id):
|
||||
raise BisyncError(
|
||||
"Bisync not initialized. Run 'bm cloud bisync --resync' to establish baseline."
|
||||
)
|
||||
|
||||
# Build rclone check command
|
||||
rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}"
|
||||
filter_path = get_bisync_filter_path()
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"check",
|
||||
str(local_path),
|
||||
rclone_remote,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
]
|
||||
|
||||
if one_way:
|
||||
cmd.append("--one-way")
|
||||
|
||||
console.print("[bold blue]Checking file integrity between local and cloud[/bold blue]")
|
||||
console.print(f"[dim]Local: {local_path}[/dim]")
|
||||
console.print(f"[dim]Remote: {rclone_remote}[/dim]")
|
||||
console.print(f"[dim]Command: {' '.join(cmd)}[/dim]")
|
||||
console.print()
|
||||
|
||||
# Run check command
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
# rclone check returns:
|
||||
# 0 = success (all files match)
|
||||
# non-zero = differences found or error
|
||||
if result.returncode == 0:
|
||||
console.print("[green]✓ All files match between local and cloud[/green]")
|
||||
return True
|
||||
else:
|
||||
console.print("[yellow]⚠ Differences found:[/yellow]")
|
||||
if result.stderr:
|
||||
console.print(result.stderr)
|
||||
if result.stdout:
|
||||
console.print(result.stdout)
|
||||
console.print("\n[dim]To sync differences, run: bm sync[/dim]")
|
||||
return False
|
||||
|
||||
except BisyncError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise BisyncError(f"Check failed: {e}") from e
|
||||
|
||||
@@ -69,16 +69,17 @@ async def create_cloud_project(project_name: str) -> CloudProjectCreateResponse:
|
||||
raise CloudUtilsError(f"Failed to create cloud project '{project_name}': {e}") from e
|
||||
|
||||
|
||||
async def sync_project(project_name: str) -> None:
|
||||
async def sync_project(project_name: str, force_full: bool = False) -> None:
|
||||
"""Trigger sync for a specific project on cloud.
|
||||
|
||||
Args:
|
||||
project_name: Name of project to sync
|
||||
force_full: If True, force a full scan bypassing watermark optimization
|
||||
"""
|
||||
try:
|
||||
from basic_memory.cli.commands.command_utils import run_sync
|
||||
|
||||
await run_sync(project=project_name)
|
||||
await run_sync(project=project_name, force_full=force_full)
|
||||
except Exception as e:
|
||||
raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Core cloud commands for Basic Memory CLI."""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
@@ -15,21 +14,16 @@ from basic_memory.cli.commands.cloud.api_client import (
|
||||
get_cloud_config,
|
||||
make_api_request,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.mount_commands import (
|
||||
mount_cloud_files,
|
||||
setup_cloud_mount,
|
||||
show_mount_status,
|
||||
unmount_cloud_files,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import (
|
||||
run_bisync,
|
||||
run_bisync_watch,
|
||||
run_check,
|
||||
setup_cloud_bisync,
|
||||
show_bisync_status,
|
||||
BisyncError,
|
||||
generate_mount_credentials,
|
||||
get_mount_info,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.rclone_config import configure_rclone_remote
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import (
|
||||
RcloneInstallError,
|
||||
install_rclone,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.rclone_config import MOUNT_PROFILES
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import BISYNC_PROFILES
|
||||
|
||||
console = Console()
|
||||
|
||||
@@ -88,18 +82,8 @@ def logout():
|
||||
|
||||
|
||||
@cloud_app.command("status")
|
||||
def status(
|
||||
bisync: bool = typer.Option(
|
||||
True,
|
||||
"--bisync/--mount",
|
||||
help="Show bisync status (default) or mount status",
|
||||
),
|
||||
) -> None:
|
||||
"""Check cloud mode status and cloud instance health.
|
||||
|
||||
Shows cloud mode status, instance health, and sync/mount status.
|
||||
Use --bisync (default) to show bisync status or --mount for mount status.
|
||||
"""
|
||||
def status() -> None:
|
||||
"""Check cloud mode status and cloud instance health."""
|
||||
# Check cloud mode
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.load_config()
|
||||
@@ -142,12 +126,7 @@ def status(
|
||||
if "timestamp" in health_data:
|
||||
console.print(f" Timestamp: {health_data['timestamp']}")
|
||||
|
||||
# Show sync/mount status based on flag
|
||||
console.print()
|
||||
if bisync:
|
||||
show_bisync_status()
|
||||
else:
|
||||
show_mount_status()
|
||||
console.print("\n[dim]To sync projects, use: bm project bisync --name <project>[/dim]")
|
||||
|
||||
except CloudAPIError as e:
|
||||
console.print(f"[red]Error checking cloud health: {e}[/red]")
|
||||
@@ -157,132 +136,60 @@ def status(
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
# Mount commands
|
||||
|
||||
|
||||
@cloud_app.command("setup")
|
||||
def setup(
|
||||
bisync: bool = typer.Option(
|
||||
True,
|
||||
"--bisync/--mount",
|
||||
help="Use bidirectional sync (recommended) or mount as network drive",
|
||||
),
|
||||
sync_dir: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--dir",
|
||||
help="Custom sync directory for bisync (default: ~/basic-memory-cloud-sync)",
|
||||
),
|
||||
) -> None:
|
||||
"""Set up cloud file access with automatic rclone installation and configuration.
|
||||
def setup() -> None:
|
||||
"""Set up cloud sync by installing rclone and configuring credentials.
|
||||
|
||||
Default: Sets up bidirectional sync (recommended).\n
|
||||
Use --mount: Sets up mount as network drive (alternative workflow).\n
|
||||
|
||||
Examples:\n
|
||||
bm cloud setup # Setup bisync (default)\n
|
||||
bm cloud setup --mount # Setup mount instead\n
|
||||
bm cloud setup --dir ~/sync # Custom bisync directory\n
|
||||
SPEC-20: Simplified to project-scoped workflow.
|
||||
After setup, use project commands for syncing:
|
||||
bm project add <name> <path> --local-path ~/projects/<name>
|
||||
bm project bisync --name <name> --resync # First time
|
||||
bm project bisync --name <name> # Subsequent syncs
|
||||
"""
|
||||
if bisync:
|
||||
setup_cloud_bisync(sync_dir=sync_dir)
|
||||
else:
|
||||
setup_cloud_mount()
|
||||
console.print("[bold blue]Basic Memory Cloud Setup[/bold blue]")
|
||||
console.print("Setting up cloud sync with rclone...\n")
|
||||
|
||||
|
||||
@cloud_app.command("mount")
|
||||
def mount(
|
||||
profile: str = typer.Option(
|
||||
"balanced", help=f"Mount profile: {', '.join(MOUNT_PROFILES.keys())}"
|
||||
),
|
||||
path: Optional[str] = typer.Option(
|
||||
None, help="Custom mount path (default: ~/basic-memory-{tenant-id})"
|
||||
),
|
||||
) -> None:
|
||||
"""Mount cloud files locally for editing."""
|
||||
try:
|
||||
mount_cloud_files(profile_name=profile)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Mount failed: {e}[/red]")
|
||||
# Step 1: Install rclone
|
||||
console.print("[blue]Step 1: Installing rclone...[/blue]")
|
||||
install_rclone()
|
||||
|
||||
# Step 2: Get tenant info
|
||||
console.print("\n[blue]Step 2: Getting tenant information...[/blue]")
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
console.print(f"[green]✓ Found tenant: {tenant_info.tenant_id}[/green]")
|
||||
|
||||
# Step 3: Generate credentials
|
||||
console.print("\n[blue]Step 3: Generating sync credentials...[/blue]")
|
||||
creds = asyncio.run(generate_mount_credentials(tenant_info.tenant_id))
|
||||
console.print("[green]✓ Generated secure credentials[/green]")
|
||||
|
||||
# Step 4: Configure rclone remote
|
||||
console.print("\n[blue]Step 4: Configuring rclone remote...[/blue]")
|
||||
configure_rclone_remote(
|
||||
access_key=creds.access_key,
|
||||
secret_key=creds.secret_key,
|
||||
)
|
||||
|
||||
console.print("\n[bold green]✓ Cloud setup completed successfully![/bold green]")
|
||||
console.print("\n[bold]Next steps:[/bold]")
|
||||
console.print("1. Add a project with local sync path:")
|
||||
console.print(" bm project add research --local-path ~/Documents/research")
|
||||
console.print("\n Or configure sync for an existing project:")
|
||||
console.print(" bm project sync-setup research ~/Documents/research")
|
||||
console.print("\n2. Preview the initial sync (recommended):")
|
||||
console.print(" bm project bisync --name research --resync --dry-run")
|
||||
console.print("\n3. If all looks good, run the actual sync:")
|
||||
console.print(" bm project bisync --name research --resync")
|
||||
console.print("\n4. Subsequent syncs (no --resync needed):")
|
||||
console.print(" bm project bisync --name research")
|
||||
console.print(
|
||||
"\n[dim]Tip: Always use --dry-run first to preview changes before syncing[/dim]"
|
||||
)
|
||||
|
||||
except (RcloneInstallError, BisyncError, CloudAPIError) as e:
|
||||
console.print(f"\n[red]Setup failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("unmount")
|
||||
def unmount() -> None:
|
||||
"""Unmount cloud files."""
|
||||
try:
|
||||
unmount_cloud_files()
|
||||
except Exception as e:
|
||||
console.print(f"[red]Unmount failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
# Bisync commands
|
||||
|
||||
|
||||
@cloud_app.command("bisync")
|
||||
def bisync(
|
||||
profile: str = typer.Option(
|
||||
"balanced", help=f"Bisync profile: {', '.join(BISYNC_PROFILES.keys())}"
|
||||
),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
resync: bool = typer.Option(False, "--resync", help="Force resync to establish new baseline"),
|
||||
watch: bool = typer.Option(False, "--watch", help="Run continuous sync in watch mode"),
|
||||
interval: int = typer.Option(60, "--interval", help="Sync interval in seconds for watch mode"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed sync output"),
|
||||
) -> None:
|
||||
"""Run bidirectional sync between local files and cloud storage.
|
||||
|
||||
Examples:
|
||||
basic-memory cloud bisync # Manual sync with balanced profile
|
||||
basic-memory cloud bisync --dry-run # Preview what would be synced
|
||||
basic-memory cloud bisync --resync # Establish new baseline
|
||||
basic-memory cloud bisync --watch # Continuous sync every 60s
|
||||
basic-memory cloud bisync --watch --interval 30 # Continuous sync every 30s
|
||||
basic-memory cloud bisync --profile safe # Use safe profile (keep conflicts)
|
||||
basic-memory cloud bisync --verbose # Show detailed file sync output
|
||||
"""
|
||||
try:
|
||||
if watch:
|
||||
run_bisync_watch(profile_name=profile, interval_seconds=interval)
|
||||
else:
|
||||
run_bisync(profile_name=profile, dry_run=dry_run, resync=resync, verbose=verbose)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Bisync failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@cloud_app.command("bisync-status")
|
||||
def bisync_status() -> None:
|
||||
"""Show current bisync status and configuration.
|
||||
|
||||
DEPRECATED: Use 'bm cloud status' instead (bisync is now the default).
|
||||
"""
|
||||
console.print(
|
||||
"[yellow]Note: 'bisync-status' is deprecated. Use 'bm cloud status' instead.[/yellow]"
|
||||
)
|
||||
console.print("[dim]Showing bisync status...[/dim]\n")
|
||||
show_bisync_status()
|
||||
|
||||
|
||||
@cloud_app.command("check")
|
||||
def check(
|
||||
one_way: bool = typer.Option(
|
||||
False,
|
||||
"--one-way",
|
||||
help="Only check for missing files on destination (faster)",
|
||||
),
|
||||
) -> None:
|
||||
"""Check file integrity between local and cloud storage using rclone check.
|
||||
|
||||
Verifies that files match between your local bisync directory and cloud storage
|
||||
without transferring any data. This is useful for validating sync integrity.
|
||||
|
||||
Examples:
|
||||
bm cloud check # Full integrity check
|
||||
bm cloud check --one-way # Faster check (missing files only)
|
||||
"""
|
||||
try:
|
||||
run_check(one_way=one_way)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Check failed: {e}[/red]")
|
||||
console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
"""Cloud mount commands for Basic Memory CLI."""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
|
||||
from basic_memory.cli.commands.cloud.api_client import CloudAPIError, make_api_request
|
||||
from basic_memory.cli.commands.cloud.rclone_config import (
|
||||
MOUNT_PROFILES,
|
||||
add_tenant_to_rclone_config,
|
||||
build_mount_command,
|
||||
cleanup_orphaned_rclone_processes,
|
||||
get_default_mount_path,
|
||||
get_rclone_processes,
|
||||
is_path_mounted,
|
||||
unmount_path,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.rclone_installer import RcloneInstallError, install_rclone
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
class MountError(Exception):
|
||||
"""Exception raised for mount-related errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
async def get_tenant_info() -> dict:
|
||||
"""Get current tenant information from cloud API."""
|
||||
try:
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
host_url = config.cloud_host.rstrip("/")
|
||||
|
||||
response = await make_api_request(method="GET", url=f"{host_url}/tenant/mount/info")
|
||||
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
raise MountError(f"Failed to get tenant info: {e}") from e
|
||||
|
||||
|
||||
async def generate_mount_credentials(tenant_id: str) -> dict:
|
||||
"""Generate scoped credentials for mounting."""
|
||||
try:
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
host_url = config.cloud_host.rstrip("/")
|
||||
|
||||
response = await make_api_request(method="POST", url=f"{host_url}/tenant/mount/credentials")
|
||||
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
raise MountError(f"Failed to generate mount credentials: {e}") from e
|
||||
|
||||
|
||||
def setup_cloud_mount() -> None:
|
||||
"""Set up cloud mount with rclone installation and configuration."""
|
||||
console.print("[bold blue]Basic Memory Cloud Setup[/bold blue]")
|
||||
console.print("Setting up local file access to your cloud tenant...\n")
|
||||
|
||||
try:
|
||||
# Step 1: Install rclone
|
||||
console.print("[blue]Step 1: Installing rclone...[/blue]")
|
||||
install_rclone()
|
||||
|
||||
# Step 2: Get tenant info
|
||||
console.print("\n[blue]Step 2: Getting tenant information...[/blue]")
|
||||
tenant_info = asyncio.run(get_tenant_info())
|
||||
|
||||
tenant_id = tenant_info.get("tenant_id")
|
||||
bucket_name = tenant_info.get("bucket_name")
|
||||
|
||||
if not tenant_id or not bucket_name:
|
||||
raise MountError("Invalid tenant information received from cloud API")
|
||||
|
||||
console.print(f"[green]✓ Found tenant: {tenant_id}[/green]")
|
||||
console.print(f"[green]✓ Bucket: {bucket_name}[/green]")
|
||||
|
||||
# Step 3: Generate mount credentials
|
||||
console.print("\n[blue]Step 3: Generating mount credentials...[/blue]")
|
||||
creds = asyncio.run(generate_mount_credentials(tenant_id))
|
||||
|
||||
access_key = creds.get("access_key")
|
||||
secret_key = creds.get("secret_key")
|
||||
|
||||
if not access_key or not secret_key:
|
||||
raise MountError("Failed to generate mount credentials")
|
||||
|
||||
console.print("[green]✓ Generated secure credentials[/green]")
|
||||
|
||||
# Step 4: Configure rclone
|
||||
console.print("\n[blue]Step 4: Configuring rclone...[/blue]")
|
||||
add_tenant_to_rclone_config(
|
||||
tenant_id=tenant_id,
|
||||
bucket_name=bucket_name,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
)
|
||||
|
||||
# Step 5: Perform initial mount
|
||||
console.print("\n[blue]Step 5: Mounting cloud files...[/blue]")
|
||||
mount_path = get_default_mount_path()
|
||||
MOUNT_PROFILES["balanced"]
|
||||
|
||||
mount_cloud_files(
|
||||
tenant_id=tenant_id,
|
||||
bucket_name=bucket_name,
|
||||
mount_path=mount_path,
|
||||
profile_name="balanced",
|
||||
)
|
||||
|
||||
console.print("\n[bold green]✓ Cloud setup completed successfully![/bold green]")
|
||||
console.print("\nYour cloud files are now accessible at:")
|
||||
console.print(f" {mount_path}")
|
||||
console.print("\nYou can now edit files locally and they will sync to the cloud!")
|
||||
console.print("\nUseful commands:")
|
||||
console.print(" basic-memory cloud mount-status # Check mount status")
|
||||
console.print(" basic-memory cloud unmount # Unmount files")
|
||||
console.print(" basic-memory cloud mount --profile fast # Remount with faster sync")
|
||||
|
||||
except (RcloneInstallError, MountError, CloudAPIError) as e:
|
||||
console.print(f"\n[red]Setup failed: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def mount_cloud_files(
|
||||
tenant_id: Optional[str] = None,
|
||||
bucket_name: Optional[str] = None,
|
||||
mount_path: Optional[Path] = None,
|
||||
profile_name: str = "balanced",
|
||||
) -> None:
|
||||
"""Mount cloud files with specified profile."""
|
||||
|
||||
try:
|
||||
# Get tenant info if not provided
|
||||
if not tenant_id or not bucket_name:
|
||||
tenant_info = asyncio.run(get_tenant_info())
|
||||
tenant_id = tenant_info.get("tenant_id")
|
||||
bucket_name = tenant_info.get("bucket_name")
|
||||
|
||||
if not tenant_id or not bucket_name:
|
||||
raise MountError("Could not determine tenant information")
|
||||
|
||||
# Set default mount path if not provided
|
||||
if not mount_path:
|
||||
mount_path = get_default_mount_path()
|
||||
|
||||
# Get mount profile
|
||||
if profile_name not in MOUNT_PROFILES:
|
||||
raise MountError(
|
||||
f"Unknown profile: {profile_name}. Available: {list(MOUNT_PROFILES.keys())}"
|
||||
)
|
||||
|
||||
profile = MOUNT_PROFILES[profile_name]
|
||||
|
||||
# Check if already mounted
|
||||
if is_path_mounted(mount_path):
|
||||
console.print(f"[yellow]Path {mount_path} is already mounted[/yellow]")
|
||||
console.print("Use 'basic-memory cloud unmount' first, or mount to a different path")
|
||||
return
|
||||
|
||||
# Create mount directory
|
||||
mount_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build and execute mount command
|
||||
mount_cmd = build_mount_command(tenant_id, bucket_name, mount_path, profile)
|
||||
|
||||
console.print(
|
||||
f"[blue]Mounting with profile '{profile_name}' ({profile.description})...[/blue]"
|
||||
)
|
||||
console.print(f"[dim]Command: {' '.join(mount_cmd)}[/dim]")
|
||||
|
||||
result = subprocess.run(mount_cmd, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
error_msg = result.stderr or "Unknown error"
|
||||
raise MountError(f"Mount command failed: {error_msg}")
|
||||
|
||||
# Wait a moment for mount to establish
|
||||
time.sleep(2)
|
||||
|
||||
# Verify mount
|
||||
if is_path_mounted(mount_path):
|
||||
console.print(f"[green]✓ Successfully mounted to {mount_path}[/green]")
|
||||
console.print(f"[green]✓ Sync profile: {profile.description}[/green]")
|
||||
else:
|
||||
raise MountError("Mount command succeeded but path is not mounted")
|
||||
|
||||
except MountError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise MountError(f"Unexpected error during mount: {e}") from e
|
||||
|
||||
|
||||
def unmount_cloud_files(tenant_id: Optional[str] = None) -> None:
|
||||
"""Unmount cloud files."""
|
||||
|
||||
try:
|
||||
# Get tenant info if not provided
|
||||
if not tenant_id:
|
||||
tenant_info = asyncio.run(get_tenant_info())
|
||||
tenant_id = tenant_info.get("tenant_id")
|
||||
|
||||
if not tenant_id:
|
||||
raise MountError("Could not determine tenant ID")
|
||||
|
||||
mount_path = get_default_mount_path()
|
||||
|
||||
if not is_path_mounted(mount_path):
|
||||
console.print(f"[yellow]Path {mount_path} is not mounted[/yellow]")
|
||||
return
|
||||
|
||||
console.print(f"[blue]Unmounting {mount_path}...[/blue]")
|
||||
|
||||
# Unmount the path
|
||||
if unmount_path(mount_path):
|
||||
console.print(f"[green]✓ Successfully unmounted {mount_path}[/green]")
|
||||
|
||||
# Clean up any orphaned rclone processes
|
||||
killed_count = cleanup_orphaned_rclone_processes()
|
||||
if killed_count > 0:
|
||||
console.print(
|
||||
f"[green]✓ Cleaned up {killed_count} orphaned rclone process(es)[/green]"
|
||||
)
|
||||
else:
|
||||
console.print(f"[red]✗ Failed to unmount {mount_path}[/red]")
|
||||
console.print("You may need to manually unmount or restart your system")
|
||||
|
||||
except MountError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise MountError(f"Unexpected error during unmount: {e}") from e
|
||||
|
||||
|
||||
def show_mount_status() -> None:
|
||||
"""Show current mount status and running processes."""
|
||||
|
||||
try:
|
||||
# Get tenant info
|
||||
tenant_info = asyncio.run(get_tenant_info())
|
||||
tenant_id = tenant_info.get("tenant_id")
|
||||
|
||||
if not tenant_id:
|
||||
console.print("[red]Could not determine tenant ID[/red]")
|
||||
return
|
||||
|
||||
mount_path = get_default_mount_path()
|
||||
|
||||
# Create status table
|
||||
table = Table(title="Cloud Mount Status", show_header=True, header_style="bold blue")
|
||||
table.add_column("Property", style="green", min_width=15)
|
||||
table.add_column("Value", style="dim", min_width=30)
|
||||
|
||||
# Check mount status
|
||||
is_mounted = is_path_mounted(mount_path)
|
||||
mount_status = "[green]✓ Mounted[/green]" if is_mounted else "[red]✗ Not mounted[/red]"
|
||||
|
||||
table.add_row("Tenant ID", tenant_id)
|
||||
table.add_row("Mount Path", str(mount_path))
|
||||
table.add_row("Status", mount_status)
|
||||
|
||||
# Get rclone processes
|
||||
processes = get_rclone_processes()
|
||||
if processes:
|
||||
table.add_row("rclone Processes", f"{len(processes)} running")
|
||||
else:
|
||||
table.add_row("rclone Processes", "None")
|
||||
|
||||
console.print(table)
|
||||
|
||||
# Show running processes details
|
||||
if processes:
|
||||
console.print("\n[bold]Running rclone processes:[/bold]")
|
||||
for proc in processes:
|
||||
console.print(f" PID {proc['pid']}: {proc['command'][:80]}...")
|
||||
|
||||
# Show mount profiles
|
||||
console.print("\n[bold]Available mount profiles:[/bold]")
|
||||
for name, profile in MOUNT_PROFILES.items():
|
||||
console.print(f" {name}: {profile.description}")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error getting mount status: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Project-scoped rclone sync commands for Basic Memory Cloud.
|
||||
|
||||
This module provides simplified, project-scoped rclone operations:
|
||||
- Each project syncs independently
|
||||
- Uses single "basic-memory-cloud" remote (not tenant-specific)
|
||||
- Balanced defaults from SPEC-8 Phase 4 testing
|
||||
- Per-project bisync state tracking
|
||||
|
||||
Replaces tenant-wide sync with project-scoped workflows.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from basic_memory.utils import normalize_project_path
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
class RcloneError(Exception):
|
||||
"""Exception raised for rclone command errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncProject:
|
||||
"""Project configured for cloud sync.
|
||||
|
||||
Attributes:
|
||||
name: Project name
|
||||
path: Cloud path (e.g., "app/data/research")
|
||||
local_sync_path: Local directory for syncing (optional)
|
||||
"""
|
||||
|
||||
name: str
|
||||
path: str
|
||||
local_sync_path: Optional[str] = None
|
||||
|
||||
|
||||
def get_bmignore_filter_path() -> Path:
|
||||
"""Get path to rclone filter file.
|
||||
|
||||
Uses ~/.basic-memory/.bmignore converted to rclone format.
|
||||
File is automatically created with default patterns on first use.
|
||||
|
||||
Returns:
|
||||
Path to rclone filter file
|
||||
"""
|
||||
# Import here to avoid circular dependency
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import (
|
||||
convert_bmignore_to_rclone_filters,
|
||||
)
|
||||
|
||||
return convert_bmignore_to_rclone_filters()
|
||||
|
||||
|
||||
def get_project_bisync_state(project_name: str) -> Path:
|
||||
"""Get path to project's bisync state directory.
|
||||
|
||||
Args:
|
||||
project_name: Name of the project
|
||||
|
||||
Returns:
|
||||
Path to bisync state directory for this project
|
||||
"""
|
||||
return Path.home() / ".basic-memory" / "bisync-state" / project_name
|
||||
|
||||
|
||||
def bisync_initialized(project_name: str) -> bool:
|
||||
"""Check if bisync has been initialized for this project.
|
||||
|
||||
Args:
|
||||
project_name: Name of the project
|
||||
|
||||
Returns:
|
||||
True if bisync state exists, False otherwise
|
||||
"""
|
||||
state_path = get_project_bisync_state(project_name)
|
||||
return state_path.exists() and any(state_path.iterdir())
|
||||
|
||||
|
||||
def get_project_remote(project: SyncProject, bucket_name: str) -> str:
|
||||
"""Build rclone remote path for project.
|
||||
|
||||
Args:
|
||||
project: Project with cloud path
|
||||
bucket_name: S3 bucket name
|
||||
|
||||
Returns:
|
||||
Remote path like "basic-memory-cloud:bucket-name/basic-memory-llc"
|
||||
|
||||
Note:
|
||||
The API returns paths like "/app/data/basic-memory-llc" because the S3 bucket
|
||||
is mounted at /app/data on the fly machine. We need to strip the /app/data/
|
||||
prefix to get the actual S3 path within the bucket.
|
||||
"""
|
||||
# Normalize path to strip /app/data/ mount point prefix
|
||||
cloud_path = normalize_project_path(project.path).lstrip("/")
|
||||
return f"basic-memory-cloud:{bucket_name}/{cloud_path}"
|
||||
|
||||
|
||||
def project_sync(
|
||||
project: SyncProject,
|
||||
bucket_name: str,
|
||||
dry_run: bool = False,
|
||||
verbose: bool = False,
|
||||
) -> bool:
|
||||
"""One-way sync: local → cloud.
|
||||
|
||||
Makes cloud identical to local using rclone sync.
|
||||
|
||||
Args:
|
||||
project: Project to sync
|
||||
bucket_name: S3 bucket name
|
||||
dry_run: Preview changes without applying
|
||||
verbose: Show detailed output
|
||||
|
||||
Returns:
|
||||
True if sync succeeded, False otherwise
|
||||
|
||||
Raises:
|
||||
RcloneError: If project has no local_sync_path configured
|
||||
"""
|
||||
if not project.local_sync_path:
|
||||
raise RcloneError(f"Project {project.name} has no local_sync_path configured")
|
||||
|
||||
local_path = Path(project.local_sync_path).expanduser()
|
||||
remote_path = get_project_remote(project, bucket_name)
|
||||
filter_path = get_bmignore_filter_path()
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"sync",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
]
|
||||
|
||||
if verbose:
|
||||
cmd.append("--verbose")
|
||||
else:
|
||||
cmd.append("--progress")
|
||||
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
result = subprocess.run(cmd, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def project_bisync(
|
||||
project: SyncProject,
|
||||
bucket_name: str,
|
||||
dry_run: bool = False,
|
||||
resync: bool = False,
|
||||
verbose: bool = False,
|
||||
) -> bool:
|
||||
"""Two-way sync: local ↔ cloud.
|
||||
|
||||
Uses rclone bisync with balanced defaults:
|
||||
- conflict_resolve: newer (auto-resolve to most recent)
|
||||
- max_delete: 25 (safety limit)
|
||||
- check_access: false (skip for performance)
|
||||
|
||||
Args:
|
||||
project: Project to sync
|
||||
bucket_name: S3 bucket name
|
||||
dry_run: Preview changes without applying
|
||||
resync: Force resync to establish new baseline
|
||||
verbose: Show detailed output
|
||||
|
||||
Returns:
|
||||
True if bisync succeeded, False otherwise
|
||||
|
||||
Raises:
|
||||
RcloneError: If project has no local_sync_path or needs --resync
|
||||
"""
|
||||
if not project.local_sync_path:
|
||||
raise RcloneError(f"Project {project.name} has no local_sync_path configured")
|
||||
|
||||
local_path = Path(project.local_sync_path).expanduser()
|
||||
remote_path = get_project_remote(project, bucket_name)
|
||||
filter_path = get_bmignore_filter_path()
|
||||
state_path = get_project_bisync_state(project.name)
|
||||
|
||||
# Ensure state directory exists
|
||||
state_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"bisync",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
"--create-empty-src-dirs",
|
||||
"--resilient",
|
||||
"--conflict-resolve=newer",
|
||||
"--max-delete=25",
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
"--workdir",
|
||||
str(state_path),
|
||||
]
|
||||
|
||||
if verbose:
|
||||
cmd.append("--verbose")
|
||||
else:
|
||||
cmd.append("--progress")
|
||||
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
|
||||
if resync:
|
||||
cmd.append("--resync")
|
||||
|
||||
# Check if first run requires resync
|
||||
if not resync and not bisync_initialized(project.name) and not dry_run:
|
||||
raise RcloneError(
|
||||
f"First bisync for {project.name} requires --resync to establish baseline.\n"
|
||||
f"Run: bm project bisync --name {project.name} --resync"
|
||||
)
|
||||
|
||||
result = subprocess.run(cmd, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def project_check(
|
||||
project: SyncProject,
|
||||
bucket_name: str,
|
||||
one_way: bool = False,
|
||||
) -> bool:
|
||||
"""Check integrity between local and cloud.
|
||||
|
||||
Verifies files match without transferring data.
|
||||
|
||||
Args:
|
||||
project: Project to check
|
||||
bucket_name: S3 bucket name
|
||||
one_way: Only check for missing files on destination (faster)
|
||||
|
||||
Returns:
|
||||
True if files match, False if differences found
|
||||
|
||||
Raises:
|
||||
RcloneError: If project has no local_sync_path configured
|
||||
"""
|
||||
if not project.local_sync_path:
|
||||
raise RcloneError(f"Project {project.name} has no local_sync_path configured")
|
||||
|
||||
local_path = Path(project.local_sync_path).expanduser()
|
||||
remote_path = get_project_remote(project, bucket_name)
|
||||
filter_path = get_bmignore_filter_path()
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"check",
|
||||
str(local_path),
|
||||
remote_path,
|
||||
"--filter-from",
|
||||
str(filter_path),
|
||||
]
|
||||
|
||||
if one_way:
|
||||
cmd.append("--one-way")
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def project_ls(
|
||||
project: SyncProject,
|
||||
bucket_name: str,
|
||||
path: Optional[str] = None,
|
||||
) -> list[str]:
|
||||
"""List files in remote project.
|
||||
|
||||
Args:
|
||||
project: Project to list files from
|
||||
bucket_name: S3 bucket name
|
||||
path: Optional subdirectory within project
|
||||
|
||||
Returns:
|
||||
List of file paths
|
||||
|
||||
Raises:
|
||||
subprocess.CalledProcessError: If rclone command fails
|
||||
"""
|
||||
remote_path = get_project_remote(project, bucket_name)
|
||||
if path:
|
||||
remote_path = f"{remote_path}/{path}"
|
||||
|
||||
cmd = ["rclone", "ls", remote_path]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
|
||||
return result.stdout.splitlines()
|
||||
@@ -1,11 +1,14 @@
|
||||
"""rclone configuration management for Basic Memory Cloud."""
|
||||
"""rclone configuration management for Basic Memory Cloud.
|
||||
|
||||
This module provides simplified rclone configuration for SPEC-20.
|
||||
Uses a single "basic-memory-cloud" remote for all operations.
|
||||
"""
|
||||
|
||||
import configparser
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
@@ -18,64 +21,6 @@ class RcloneConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class RcloneMountProfile:
|
||||
"""Mount profile with optimized settings."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
cache_time: str,
|
||||
poll_interval: str,
|
||||
attr_timeout: str,
|
||||
write_back: str,
|
||||
description: str,
|
||||
extra_args: Optional[List[str]] = None,
|
||||
):
|
||||
self.name = name
|
||||
self.cache_time = cache_time
|
||||
self.poll_interval = poll_interval
|
||||
self.attr_timeout = attr_timeout
|
||||
self.write_back = write_back
|
||||
self.description = description
|
||||
self.extra_args = extra_args or []
|
||||
|
||||
|
||||
# Mount profiles based on SPEC-7 Phase 4 testing
|
||||
MOUNT_PROFILES = {
|
||||
"fast": RcloneMountProfile(
|
||||
name="fast",
|
||||
cache_time="5s",
|
||||
poll_interval="3s",
|
||||
attr_timeout="3s",
|
||||
write_back="1s",
|
||||
description="Ultra-fast development (5s sync, higher bandwidth)",
|
||||
),
|
||||
"balanced": RcloneMountProfile(
|
||||
name="balanced",
|
||||
cache_time="10s",
|
||||
poll_interval="5s",
|
||||
attr_timeout="5s",
|
||||
write_back="2s",
|
||||
description="Fast development (10-15s sync, recommended)",
|
||||
),
|
||||
"safe": RcloneMountProfile(
|
||||
name="safe",
|
||||
cache_time="15s",
|
||||
poll_interval="10s",
|
||||
attr_timeout="10s",
|
||||
write_back="5s",
|
||||
description="Conflict-aware mount with backup",
|
||||
extra_args=[
|
||||
"--conflict-suffix",
|
||||
".conflict-{DateTimeExt}",
|
||||
"--backup-dir",
|
||||
"~/.basic-memory/conflicts",
|
||||
"--track-renames",
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_rclone_config_path() -> Path:
|
||||
"""Get the path to rclone configuration file."""
|
||||
config_dir = Path.home() / ".config" / "rclone"
|
||||
@@ -116,173 +61,50 @@ def save_rclone_config(config: configparser.ConfigParser) -> None:
|
||||
console.print(f"[dim]Updated rclone config: {config_path}[/dim]")
|
||||
|
||||
|
||||
def add_tenant_to_rclone_config(
|
||||
tenant_id: str,
|
||||
bucket_name: str,
|
||||
def configure_rclone_remote(
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
endpoint: str = "https://fly.storage.tigris.dev",
|
||||
region: str = "auto",
|
||||
) -> str:
|
||||
"""Add tenant configuration to rclone config file."""
|
||||
"""Configure single rclone remote named 'basic-memory-cloud'.
|
||||
|
||||
This is the simplified approach from SPEC-20 that uses one remote
|
||||
for all Basic Memory cloud operations (not tenant-specific).
|
||||
|
||||
Args:
|
||||
access_key: S3 access key ID
|
||||
secret_key: S3 secret access key
|
||||
endpoint: S3-compatible endpoint URL
|
||||
region: S3 region (default: auto)
|
||||
|
||||
Returns:
|
||||
The remote name: "basic-memory-cloud"
|
||||
"""
|
||||
# Backup existing config
|
||||
backup_rclone_config()
|
||||
|
||||
# Load existing config
|
||||
config = load_rclone_config()
|
||||
|
||||
# Create section name
|
||||
section_name = f"basic-memory-{tenant_id}"
|
||||
# Single remote name (not tenant-specific)
|
||||
REMOTE_NAME = "basic-memory-cloud"
|
||||
|
||||
# Add/update the tenant section
|
||||
if not config.has_section(section_name):
|
||||
config.add_section(section_name)
|
||||
|
||||
config.set(section_name, "type", "s3")
|
||||
config.set(section_name, "provider", "Other")
|
||||
config.set(section_name, "access_key_id", access_key)
|
||||
config.set(section_name, "secret_access_key", secret_key)
|
||||
config.set(section_name, "endpoint", endpoint)
|
||||
config.set(section_name, "region", region)
|
||||
# Add/update the remote section
|
||||
if not config.has_section(REMOTE_NAME):
|
||||
config.add_section(REMOTE_NAME)
|
||||
|
||||
config.set(REMOTE_NAME, "type", "s3")
|
||||
config.set(REMOTE_NAME, "provider", "Other")
|
||||
config.set(REMOTE_NAME, "access_key_id", access_key)
|
||||
config.set(REMOTE_NAME, "secret_access_key", secret_key)
|
||||
config.set(REMOTE_NAME, "endpoint", endpoint)
|
||||
config.set(REMOTE_NAME, "region", region)
|
||||
# Prevent unnecessary encoding of filenames (only encode slashes and invalid UTF-8)
|
||||
# This prevents files with spaces like "Hello World.md" from being quoted
|
||||
config.set(REMOTE_NAME, "encoding", "Slash,InvalidUtf8")
|
||||
# Save updated config
|
||||
save_rclone_config(config)
|
||||
|
||||
console.print(f"[green]✓ Added tenant {tenant_id} to rclone config[/green]")
|
||||
return section_name
|
||||
|
||||
|
||||
def remove_tenant_from_rclone_config(tenant_id: str) -> bool:
|
||||
"""Remove tenant configuration from rclone config."""
|
||||
config = load_rclone_config()
|
||||
section_name = f"basic-memory-{tenant_id}"
|
||||
|
||||
if config.has_section(section_name):
|
||||
backup_rclone_config()
|
||||
config.remove_section(section_name)
|
||||
save_rclone_config(config)
|
||||
console.print(f"[green]✓ Removed tenant {tenant_id} from rclone config[/green]")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_default_mount_path() -> Path:
|
||||
"""Get default mount path (fixed location per SPEC-9).
|
||||
|
||||
Returns:
|
||||
Fixed mount path: ~/basic-memory-cloud/
|
||||
"""
|
||||
return Path.home() / "basic-memory-cloud"
|
||||
|
||||
|
||||
def build_mount_command(
|
||||
tenant_id: str, bucket_name: str, mount_path: Path, profile: RcloneMountProfile
|
||||
) -> List[str]:
|
||||
"""Build rclone mount command with optimized settings."""
|
||||
|
||||
rclone_remote = f"basic-memory-{tenant_id}:{bucket_name}"
|
||||
|
||||
cmd = [
|
||||
"rclone",
|
||||
"nfsmount",
|
||||
rclone_remote,
|
||||
str(mount_path),
|
||||
"--vfs-cache-mode",
|
||||
"writes",
|
||||
"--dir-cache-time",
|
||||
profile.cache_time,
|
||||
"--vfs-cache-poll-interval",
|
||||
profile.poll_interval,
|
||||
"--attr-timeout",
|
||||
profile.attr_timeout,
|
||||
"--vfs-write-back",
|
||||
profile.write_back,
|
||||
"--daemon",
|
||||
]
|
||||
|
||||
# Add profile-specific extra arguments
|
||||
cmd.extend(profile.extra_args)
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
def is_path_mounted(mount_path: Path) -> bool:
|
||||
"""Check if a path is currently mounted."""
|
||||
if not mount_path.exists():
|
||||
return False
|
||||
|
||||
try:
|
||||
# Check if mount point is actually mounted by looking for mount table entry
|
||||
result = subprocess.run(["mount"], capture_output=True, text=True, check=False)
|
||||
|
||||
if result.returncode == 0:
|
||||
# Look for our mount path in mount output
|
||||
mount_str = str(mount_path.resolve())
|
||||
return mount_str in result.stdout
|
||||
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def get_rclone_processes() -> List[Dict[str, str]]:
|
||||
"""Get list of running rclone processes."""
|
||||
try:
|
||||
# Use ps to find rclone processes
|
||||
result = subprocess.run(
|
||||
["ps", "-eo", "pid,args"], capture_output=True, text=True, check=False
|
||||
)
|
||||
|
||||
processes = []
|
||||
if result.returncode == 0:
|
||||
for line in result.stdout.split("\n"):
|
||||
if "rclone" in line and "basic-memory" in line:
|
||||
parts = line.strip().split(None, 1)
|
||||
if len(parts) >= 2:
|
||||
processes.append({"pid": parts[0], "command": parts[1]})
|
||||
|
||||
return processes
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def kill_rclone_process(pid: str) -> bool:
|
||||
"""Kill a specific rclone process."""
|
||||
try:
|
||||
subprocess.run(["kill", pid], check=True)
|
||||
console.print(f"[green]✓ Killed rclone process {pid}[/green]")
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
console.print(f"[red]✗ Failed to kill rclone process {pid}[/red]")
|
||||
return False
|
||||
|
||||
|
||||
def unmount_path(mount_path: Path) -> bool:
|
||||
"""Unmount a mounted path."""
|
||||
if not is_path_mounted(mount_path):
|
||||
return True
|
||||
|
||||
try:
|
||||
subprocess.run(["umount", str(mount_path)], check=True)
|
||||
console.print(f"[green]✓ Unmounted {mount_path}[/green]")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
console.print(f"[red]✗ Failed to unmount {mount_path}: {e}[/red]")
|
||||
return False
|
||||
|
||||
|
||||
def cleanup_orphaned_rclone_processes() -> int:
|
||||
"""Clean up orphaned rclone processes for basic-memory."""
|
||||
processes = get_rclone_processes()
|
||||
killed_count = 0
|
||||
|
||||
for proc in processes:
|
||||
console.print(
|
||||
f"[yellow]Found rclone process: {proc['pid']} - {proc['command'][:80]}...[/yellow]"
|
||||
)
|
||||
if kill_rclone_process(proc["pid"]):
|
||||
killed_count += 1
|
||||
|
||||
return killed_count
|
||||
console.print(f"[green]✓ Configured rclone remote: {REMOTE_NAME}[/green]")
|
||||
return REMOTE_NAME
|
||||
|
||||
@@ -112,10 +112,11 @@ def upload(
|
||||
console.print(f"[green]✅ Successfully uploaded to '{project}'[/green]")
|
||||
|
||||
# Sync project if requested (skip on dry run)
|
||||
# Force full scan after bisync to ensure database is up-to-date with synced files
|
||||
if sync and not dry_run:
|
||||
console.print(f"[blue]Syncing project '{project}'...[/blue]")
|
||||
try:
|
||||
await sync_project(project)
|
||||
await sync_project(project, force_full=True)
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Sync failed: {e}[/yellow]")
|
||||
console.print("[dim]Files uploaded but may not be indexed yet[/dim]")
|
||||
|
||||
@@ -16,13 +16,21 @@ from basic_memory.schemas import ProjectInfoResponse
|
||||
console = Console()
|
||||
|
||||
|
||||
async def run_sync(project: Optional[str] = None):
|
||||
"""Run sync operation via API endpoint."""
|
||||
async def run_sync(project: Optional[str] = None, force_full: bool = False):
|
||||
"""Run sync operation via API endpoint.
|
||||
|
||||
Args:
|
||||
project: Optional project name
|
||||
force_full: If True, force a full scan bypassing watermark optimization
|
||||
"""
|
||||
|
||||
try:
|
||||
async with get_client() as client:
|
||||
project_item = await get_active_project(client, project, None)
|
||||
response = await call_post(client, f"{project_item.project_url}/project/sync")
|
||||
url = f"{project_item.project_url}/project/sync"
|
||||
if force_full:
|
||||
url += "?force_full=true"
|
||||
response = await call_post(client, url)
|
||||
data = response.json()
|
||||
console.print(f"[green]✓ {data['message']}[/green]")
|
||||
except (ToolError, ValueError) as e:
|
||||
|
||||
@@ -37,8 +37,8 @@ def reset(
|
||||
logger.info("Database reset complete")
|
||||
|
||||
if reindex:
|
||||
# Import and run sync
|
||||
from basic_memory.cli.commands.sync import sync
|
||||
# Run database sync directly
|
||||
from basic_memory.cli.commands.command_utils import run_sync
|
||||
|
||||
logger.info("Rebuilding search index from filesystem...")
|
||||
sync(watch=False) # pyright: ignore
|
||||
asyncio.run(run_sync(project=None))
|
||||
|
||||
@@ -22,9 +22,20 @@ from basic_memory.mcp.tools.utils import call_post
|
||||
from basic_memory.schemas.project_info import ProjectStatusResponse
|
||||
from basic_memory.mcp.tools.utils import call_delete
|
||||
from basic_memory.mcp.tools.utils import call_put
|
||||
from basic_memory.utils import generate_permalink
|
||||
from basic_memory.utils import generate_permalink, normalize_project_path
|
||||
from basic_memory.mcp.tools.utils import call_patch
|
||||
|
||||
# Import rclone commands for project sync
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import (
|
||||
SyncProject,
|
||||
RcloneError,
|
||||
project_sync,
|
||||
project_bisync,
|
||||
project_check,
|
||||
project_ls,
|
||||
)
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import get_mount_info
|
||||
|
||||
console = Console()
|
||||
|
||||
# Create a project subcommand
|
||||
@@ -51,15 +62,41 @@ def list_projects() -> None:
|
||||
|
||||
try:
|
||||
result = asyncio.run(_list_projects())
|
||||
config = ConfigManager().config
|
||||
|
||||
table = Table(title="Basic Memory Projects")
|
||||
table.add_column("Name", style="cyan")
|
||||
table.add_column("Path", style="green")
|
||||
table.add_column("Default", style="magenta")
|
||||
|
||||
# Add Local Path column if in cloud mode
|
||||
if config.cloud_mode_enabled:
|
||||
table.add_column("Local Path", style="yellow", no_wrap=True, overflow="fold")
|
||||
|
||||
# Show Default column in local mode or if default_project_mode is enabled in cloud mode
|
||||
show_default_column = not config.cloud_mode_enabled or config.default_project_mode
|
||||
if show_default_column:
|
||||
table.add_column("Default", style="magenta")
|
||||
|
||||
for project in result.projects:
|
||||
is_default = "✓" if project.is_default else ""
|
||||
table.add_row(project.name, format_path(project.path), is_default)
|
||||
normalized_path = normalize_project_path(project.path)
|
||||
|
||||
# Build row based on mode
|
||||
row = [project.name, format_path(normalized_path)]
|
||||
|
||||
# Add local path if in cloud mode
|
||||
if config.cloud_mode_enabled:
|
||||
local_path = ""
|
||||
if project.name in config.cloud_projects:
|
||||
local_path = config.cloud_projects[project.name].local_path or ""
|
||||
local_path = format_path(local_path)
|
||||
row.append(local_path)
|
||||
|
||||
# Add default indicator if showing default column
|
||||
if show_default_column:
|
||||
row.append(is_default)
|
||||
|
||||
table.add_row(*row)
|
||||
|
||||
console.print(table)
|
||||
except Exception as e:
|
||||
@@ -73,20 +110,38 @@ def add_project(
|
||||
path: str = typer.Argument(
|
||||
None, help="Path to the project directory (required for local mode)"
|
||||
),
|
||||
local_path: str = typer.Option(
|
||||
None, "--local-path", help="Local sync path for cloud mode (optional)"
|
||||
),
|
||||
set_default: bool = typer.Option(False, "--default", help="Set as default project"),
|
||||
) -> None:
|
||||
"""Add a new project.
|
||||
|
||||
For cloud mode: only name is required
|
||||
For local mode: both name and path are required
|
||||
Cloud mode examples:\n
|
||||
bm project add research # No local sync\n
|
||||
bm project add research --local-path ~/docs # With local sync\n
|
||||
|
||||
Local mode example:\n
|
||||
bm project add research ~/Documents/research
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
|
||||
# Resolve local sync path early (needed for both cloud and local mode)
|
||||
local_sync_path: str | None = None
|
||||
if local_path:
|
||||
local_sync_path = Path(os.path.abspath(os.path.expanduser(local_path))).as_posix()
|
||||
|
||||
if config.cloud_mode_enabled:
|
||||
# Cloud mode: path not needed (auto-generated from name)
|
||||
# Cloud mode: path auto-generated from name, local sync is optional
|
||||
|
||||
async def _add_project():
|
||||
async with get_client() as client:
|
||||
data = {"name": name, "path": generate_permalink(name), "set_default": set_default}
|
||||
data = {
|
||||
"name": name,
|
||||
"path": generate_permalink(name),
|
||||
"local_sync_path": local_sync_path,
|
||||
"set_default": set_default,
|
||||
}
|
||||
response = await call_post(client, "/projects/projects", json=data)
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
else:
|
||||
@@ -107,13 +162,85 @@ def add_project(
|
||||
try:
|
||||
result = asyncio.run(_add_project())
|
||||
console.print(f"[green]{result.message}[/green]")
|
||||
|
||||
# Save local sync path to config if in cloud mode
|
||||
if config.cloud_mode_enabled and local_sync_path:
|
||||
from basic_memory.config import CloudProjectConfig
|
||||
|
||||
# Create local directory if it doesn't exist
|
||||
local_dir = Path(local_sync_path)
|
||||
local_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Update config with sync path
|
||||
config.cloud_projects[name] = CloudProjectConfig(
|
||||
local_path=local_sync_path,
|
||||
last_sync=None,
|
||||
bisync_initialized=False,
|
||||
)
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
console.print(f"\n[green]✓ Local sync path configured: {local_sync_path}[/green]")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error adding project: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Display usage hint
|
||||
console.print("\nTo use this project:")
|
||||
console.print(f" basic-memory --project={name} <command>")
|
||||
|
||||
@project_app.command("sync-setup")
|
||||
def setup_project_sync(
|
||||
name: str = typer.Argument(..., help="Project name"),
|
||||
local_path: str = typer.Argument(..., help="Local sync directory"),
|
||||
) -> None:
|
||||
"""Configure local sync for an existing cloud project.
|
||||
|
||||
Example:
|
||||
bm project sync-setup research ~/Documents/research
|
||||
"""
|
||||
config_manager = ConfigManager()
|
||||
config = config_manager.config
|
||||
|
||||
if not config.cloud_mode_enabled:
|
||||
console.print("[red]Error: sync-setup only available in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
async def _verify_project_exists():
|
||||
"""Verify the project exists on cloud by listing all projects."""
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/projects/projects")
|
||||
project_list = response.json()
|
||||
project_names = [p["name"] for p in project_list["projects"]]
|
||||
if name not in project_names:
|
||||
raise ValueError(f"Project '{name}' not found on cloud")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Verify project exists on cloud
|
||||
asyncio.run(_verify_project_exists())
|
||||
|
||||
# Resolve and create local path
|
||||
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path)))
|
||||
resolved_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Update local config with sync path
|
||||
from basic_memory.config import CloudProjectConfig
|
||||
|
||||
config.cloud_projects[name] = CloudProjectConfig(
|
||||
local_path=resolved_path.as_posix(),
|
||||
last_sync=None,
|
||||
bisync_initialized=False,
|
||||
)
|
||||
config_manager.save_config(config)
|
||||
|
||||
console.print(f"[green]✓ Sync configured for project '{name}'[/green]")
|
||||
console.print(f"\nLocal sync path: {resolved_path}")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error configuring sync: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("remove")
|
||||
@@ -134,16 +261,59 @@ def remove_project(
|
||||
return ProjectStatusResponse.model_validate(response.json())
|
||||
|
||||
try:
|
||||
# Get config to check for local sync path and bisync state
|
||||
config = ConfigManager().config
|
||||
local_path = None
|
||||
has_bisync_state = False
|
||||
|
||||
if config.cloud_mode_enabled and name in config.cloud_projects:
|
||||
local_path = config.cloud_projects[name].local_path
|
||||
|
||||
# Check for bisync state
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
|
||||
|
||||
bisync_state_path = get_project_bisync_state(name)
|
||||
has_bisync_state = bisync_state_path.exists()
|
||||
|
||||
# Remove project from cloud/API
|
||||
result = asyncio.run(_remove_project())
|
||||
console.print(f"[green]{result.message}[/green]")
|
||||
|
||||
# Clean up local sync directory if it exists and delete_notes is True
|
||||
if delete_notes and local_path:
|
||||
local_dir = Path(local_path)
|
||||
if local_dir.exists():
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(local_dir)
|
||||
console.print(f"[green]✓ Removed local sync directory: {local_path}[/green]")
|
||||
|
||||
# Clean up bisync state if it exists
|
||||
if has_bisync_state:
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
|
||||
import shutil
|
||||
|
||||
bisync_state_path = get_project_bisync_state(name)
|
||||
if bisync_state_path.exists():
|
||||
shutil.rmtree(bisync_state_path)
|
||||
console.print("[green]✓ Removed bisync state[/green]")
|
||||
|
||||
# Clean up cloud_projects config entry
|
||||
if config.cloud_mode_enabled and name in config.cloud_projects:
|
||||
del config.cloud_projects[name]
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
# Show informative message if files were not deleted
|
||||
if not delete_notes:
|
||||
if local_path:
|
||||
console.print(f"[yellow]Note: Local files remain at {local_path}[/yellow]")
|
||||
else:
|
||||
console.print("[yellow]Note: Cloud project files have not been deleted.[/yellow]")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error removing project: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Show this message only if files were not deleted
|
||||
if not delete_notes:
|
||||
console.print("[yellow]Note: The project files have not been deleted from disk.[/yellow]")
|
||||
|
||||
|
||||
@project_app.command("default")
|
||||
def set_default_project(
|
||||
@@ -248,6 +418,349 @@ def move_project(
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("sync")
|
||||
def sync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to sync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""One-way sync: local → cloud (make cloud identical to local).
|
||||
|
||||
Example:
|
||||
bm project sync --name research
|
||||
bm project sync --name research --dry-run
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
if not config.cloud_mode_enabled:
|
||||
console.print("[red]Error: sync only available in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
async def _get_project():
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/projects/projects")
|
||||
projects_list = ProjectList.model_validate(response.json())
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
project_data = asyncio.run(_get_project())
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get local_sync_path from cloud_projects config
|
||||
local_sync_path = None
|
||||
if name in config.cloud_projects:
|
||||
local_sync_path = config.cloud_projects[name].local_path
|
||||
|
||||
if not local_sync_path:
|
||||
console.print(f"[red]Error: Project '{name}' has no local_sync_path configured[/red]")
|
||||
console.print(f"\nConfigure sync with: bm project sync-setup {name} ~/path/to/local")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Create SyncProject
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
|
||||
# Run sync
|
||||
console.print(f"[blue]Syncing {name} (local → cloud)...[/blue]")
|
||||
success = project_sync(sync_project, bucket_name, dry_run=dry_run, verbose=verbose)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]✓ {name} synced successfully[/green]")
|
||||
|
||||
# Trigger database sync if not a dry run
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
permalink = generate_permalink(name)
|
||||
response = await call_post(client, f"/{permalink}/project/sync", json={})
|
||||
return response.json()
|
||||
|
||||
try:
|
||||
result = asyncio.run(_trigger_db_sync())
|
||||
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
|
||||
else:
|
||||
console.print(f"[red]✗ {name} sync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Sync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("bisync")
|
||||
def bisync_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to bisync"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Preview changes without syncing"),
|
||||
resync: bool = typer.Option(False, "--resync", help="Force new baseline"),
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
|
||||
) -> None:
|
||||
"""Two-way sync: local ↔ cloud (bidirectional sync).
|
||||
|
||||
Examples:
|
||||
bm project bisync --name research --resync # First time
|
||||
bm project bisync --name research # Subsequent syncs
|
||||
bm project bisync --name research --dry-run # Preview changes
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
if not config.cloud_mode_enabled:
|
||||
console.print("[red]Error: bisync only available in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
async def _get_project():
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/projects/projects")
|
||||
projects_list = ProjectList.model_validate(response.json())
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
project_data = asyncio.run(_get_project())
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get local_sync_path from cloud_projects config
|
||||
local_sync_path = None
|
||||
if name in config.cloud_projects:
|
||||
local_sync_path = config.cloud_projects[name].local_path
|
||||
|
||||
if not local_sync_path:
|
||||
console.print(f"[red]Error: Project '{name}' has no local_sync_path configured[/red]")
|
||||
console.print(f"\nConfigure sync with: bm project sync-setup {name} ~/path/to/local")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Create SyncProject
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
|
||||
# Run bisync
|
||||
console.print(f"[blue]Bisync {name} (local ↔ cloud)...[/blue]")
|
||||
success = project_bisync(
|
||||
sync_project, bucket_name, dry_run=dry_run, resync=resync, verbose=verbose
|
||||
)
|
||||
|
||||
if success:
|
||||
console.print(f"[green]✓ {name} bisync completed successfully[/green]")
|
||||
|
||||
# Update config
|
||||
config.cloud_projects[name].last_sync = datetime.now()
|
||||
config.cloud_projects[name].bisync_initialized = True
|
||||
ConfigManager().save_config(config)
|
||||
|
||||
# Trigger database sync if not a dry run
|
||||
if not dry_run:
|
||||
|
||||
async def _trigger_db_sync():
|
||||
async with get_client() as client:
|
||||
permalink = generate_permalink(name)
|
||||
response = await call_post(client, f"/{permalink}/project/sync", json={})
|
||||
return response.json()
|
||||
|
||||
try:
|
||||
result = asyncio.run(_trigger_db_sync())
|
||||
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
|
||||
else:
|
||||
console.print(f"[red]✗ {name} bisync failed[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Bisync error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("check")
|
||||
def check_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to check"),
|
||||
one_way: bool = typer.Option(False, "--one-way", help="Check one direction only (faster)"),
|
||||
) -> None:
|
||||
"""Verify file integrity between local and cloud.
|
||||
|
||||
Example:
|
||||
bm project check --name research
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
if not config.cloud_mode_enabled:
|
||||
console.print("[red]Error: check only available in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
async def _get_project():
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/projects/projects")
|
||||
projects_list = ProjectList.model_validate(response.json())
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
project_data = asyncio.run(_get_project())
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get local_sync_path from cloud_projects config
|
||||
local_sync_path = None
|
||||
if name in config.cloud_projects:
|
||||
local_sync_path = config.cloud_projects[name].local_path
|
||||
|
||||
if not local_sync_path:
|
||||
console.print(f"[red]Error: Project '{name}' has no local_sync_path configured[/red]")
|
||||
console.print(f"\nConfigure sync with: bm project sync-setup {name} ~/path/to/local")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Create SyncProject
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
local_sync_path=local_sync_path,
|
||||
)
|
||||
|
||||
# Run check
|
||||
console.print(f"[blue]Checking {name} integrity...[/blue]")
|
||||
match = project_check(sync_project, bucket_name, one_way=one_way)
|
||||
|
||||
if match:
|
||||
console.print(f"[green]✓ {name} files match[/green]")
|
||||
else:
|
||||
console.print(f"[yellow]⚠ {name} has differences[/yellow]")
|
||||
|
||||
except RcloneError as e:
|
||||
console.print(f"[red]Check error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("bisync-reset")
|
||||
def bisync_reset(
|
||||
name: str = typer.Argument(..., help="Project name to reset bisync state for"),
|
||||
) -> None:
|
||||
"""Clear bisync state for a project.
|
||||
|
||||
This removes the bisync metadata files, forcing a fresh --resync on next bisync.
|
||||
Useful when bisync gets into an inconsistent state or when remote path changes.
|
||||
"""
|
||||
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
|
||||
import shutil
|
||||
|
||||
try:
|
||||
state_path = get_project_bisync_state(name)
|
||||
|
||||
if not state_path.exists():
|
||||
console.print(f"[yellow]No bisync state found for project '{name}'[/yellow]")
|
||||
return
|
||||
|
||||
# Remove the entire state directory
|
||||
shutil.rmtree(state_path)
|
||||
console.print(f"[green]✓ Cleared bisync state for project '{name}'[/green]")
|
||||
console.print("\nNext steps:")
|
||||
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
|
||||
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error clearing bisync state: {str(e)}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("ls")
|
||||
def ls_project_command(
|
||||
name: str = typer.Option(..., "--name", help="Project name to list files from"),
|
||||
path: str = typer.Argument(None, help="Path within project (optional)"),
|
||||
) -> None:
|
||||
"""List files in remote project.
|
||||
|
||||
Examples:
|
||||
bm project ls --name research
|
||||
bm project ls --name research subfolder
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
if not config.cloud_mode_enabled:
|
||||
console.print("[red]Error: ls only available in cloud mode[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Get tenant info for bucket name
|
||||
tenant_info = asyncio.run(get_mount_info())
|
||||
bucket_name = tenant_info.bucket_name
|
||||
|
||||
# Get project info
|
||||
async def _get_project():
|
||||
async with get_client() as client:
|
||||
response = await call_get(client, "/projects/projects")
|
||||
projects_list = ProjectList.model_validate(response.json())
|
||||
for proj in projects_list.projects:
|
||||
if generate_permalink(proj.name) == generate_permalink(name):
|
||||
return proj
|
||||
return None
|
||||
|
||||
project_data = asyncio.run(_get_project())
|
||||
if not project_data:
|
||||
console.print(f"[red]Error: Project '{name}' not found[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Create SyncProject (local_sync_path not needed for ls)
|
||||
sync_project = SyncProject(
|
||||
name=project_data.name,
|
||||
path=normalize_project_path(project_data.path),
|
||||
)
|
||||
|
||||
# List files
|
||||
files = project_ls(sync_project, bucket_name, path=path)
|
||||
|
||||
if files:
|
||||
console.print(f"\n[bold]Files in {name}" + (f"/{path}" if path else "") + ":[/bold]")
|
||||
for file in files:
|
||||
console.print(f" {file}")
|
||||
console.print(f"\n[dim]Total: {len(files)} files[/dim]")
|
||||
else:
|
||||
console.print(
|
||||
f"[yellow]No files found in {name}" + (f"/{path}" if path else "") + "[/yellow]"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/red]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@project_app.command("info")
|
||||
def display_project_info(
|
||||
name: str = typer.Argument(..., help="Name of the project"),
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
"""Command module for basic-memory sync operations."""
|
||||
|
||||
import asyncio
|
||||
from typing import Annotated, Optional
|
||||
|
||||
import typer
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.command_utils import run_sync
|
||||
from basic_memory.config import ConfigManager
|
||||
|
||||
|
||||
@app.command()
|
||||
def sync(
|
||||
project: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="The project name."),
|
||||
] = None,
|
||||
watch: Annotated[
|
||||
bool,
|
||||
typer.Option("--watch", help="Run continuous sync (cloud mode only)"),
|
||||
] = False,
|
||||
interval: Annotated[
|
||||
int,
|
||||
typer.Option("--interval", help="Sync interval in seconds for watch mode (default: 60)"),
|
||||
] = 60,
|
||||
) -> None:
|
||||
"""Sync knowledge files with the database.
|
||||
|
||||
In local mode: Scans filesystem and updates database.
|
||||
In cloud mode: Runs bidirectional file sync (bisync) then updates database.
|
||||
|
||||
Examples:
|
||||
bm sync # One-time sync
|
||||
bm sync --watch # Continuous sync every 60s
|
||||
bm sync --watch --interval 30 # Continuous sync every 30s
|
||||
"""
|
||||
config = ConfigManager().config
|
||||
|
||||
if config.cloud_mode_enabled:
|
||||
# Cloud mode: run bisync which includes database sync
|
||||
from basic_memory.cli.commands.cloud.bisync_commands import run_bisync, run_bisync_watch
|
||||
|
||||
try:
|
||||
if watch:
|
||||
run_bisync_watch(interval_seconds=interval)
|
||||
else:
|
||||
run_bisync()
|
||||
except Exception:
|
||||
raise typer.Exit(1)
|
||||
else:
|
||||
# Local mode: just database sync
|
||||
if watch:
|
||||
typer.echo(
|
||||
"Error: --watch is only available in cloud mode. Run 'bm cloud login' first."
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
|
||||
asyncio.run(run_sync(project))
|
||||
@@ -13,7 +13,6 @@ from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
mcp,
|
||||
project,
|
||||
status,
|
||||
sync,
|
||||
tool,
|
||||
)
|
||||
|
||||
|
||||
+26
-10
@@ -3,11 +3,12 @@
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Literal, Optional, List, Tuple
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
import basic_memory
|
||||
@@ -39,6 +40,22 @@ class ProjectConfig:
|
||||
return f"/{generate_permalink(self.name)}"
|
||||
|
||||
|
||||
class CloudProjectConfig(BaseModel):
|
||||
"""Sync configuration for a cloud project.
|
||||
|
||||
This tracks the local working directory and sync state for a project
|
||||
that is synced with Basic Memory Cloud.
|
||||
"""
|
||||
|
||||
local_path: str = Field(description="Local working directory path for this cloud project")
|
||||
last_sync: Optional[datetime] = Field(
|
||||
default=None, description="Timestamp of last successful sync operation"
|
||||
)
|
||||
bisync_initialized: bool = Field(
|
||||
default=False, description="Whether rclone bisync baseline has been established"
|
||||
)
|
||||
|
||||
|
||||
class BasicMemoryConfig(BaseSettings):
|
||||
"""Pydantic model for Basic Memory global configuration."""
|
||||
|
||||
@@ -138,6 +155,11 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Enable cloud mode - all requests go to cloud instead of local (config file value)",
|
||||
)
|
||||
|
||||
cloud_projects: Dict[str, CloudProjectConfig] = Field(
|
||||
default_factory=dict,
|
||||
description="Cloud project sync configuration mapping project names to their local paths and sync state",
|
||||
)
|
||||
|
||||
@property
|
||||
def cloud_mode_enabled(self) -> bool:
|
||||
"""Check if cloud mode is enabled.
|
||||
@@ -154,14 +176,6 @@ class BasicMemoryConfig(BaseSettings):
|
||||
# Fall back to config file value
|
||||
return self.cloud_mode
|
||||
|
||||
bisync_config: Dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"profile": "balanced",
|
||||
"sync_dir": str(Path.home() / "basic-memory-cloud-sync"),
|
||||
},
|
||||
description="Bisync configuration for cloud sync",
|
||||
)
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="BASIC_MEMORY_",
|
||||
extra="ignore",
|
||||
@@ -427,7 +441,9 @@ def get_project_config(project_name: Optional[str] = None) -> ProjectConfig:
|
||||
def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None:
|
||||
"""Save configuration to file."""
|
||||
try:
|
||||
file_path.write_text(json.dumps(config.model_dump(), indent=2))
|
||||
# Use model_dump with mode='json' to serialize datetime objects properly
|
||||
config_dict = config.model_dump(mode="json")
|
||||
file_path.write_text(json.dumps(config_dict, indent=2))
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to save config: {e}")
|
||||
|
||||
|
||||
@@ -234,16 +234,26 @@ class ProjectService:
|
||||
if not self.repository: # pragma: no cover
|
||||
raise ValueError("Repository is required for remove_project")
|
||||
|
||||
# Get project path before removing from config
|
||||
# Get project from database first
|
||||
project = await self.get_project(name)
|
||||
project_path = project.path if project else None
|
||||
if not project:
|
||||
raise ValueError(f"Project '{name}' not found")
|
||||
|
||||
# First remove from config (this will validate the project exists and is not default)
|
||||
self.config_manager.remove_project(name)
|
||||
project_path = project.path
|
||||
|
||||
# Then remove from database using robust lookup
|
||||
if project:
|
||||
await self.repository.delete(project.id)
|
||||
# Check if project is default (in cloud mode, check database; in local mode, check config)
|
||||
if project.is_default or name == self.config_manager.config.default_project:
|
||||
raise ValueError(f"Cannot remove the default project '{name}'")
|
||||
|
||||
# Remove from config if it exists there (may not exist in cloud mode)
|
||||
try:
|
||||
self.config_manager.remove_project(name)
|
||||
except ValueError:
|
||||
# Project not in config - that's OK in cloud mode, continue with database deletion
|
||||
logger.debug(f"Project '{name}' not found in config, removing from database only")
|
||||
|
||||
# Remove from database
|
||||
await self.repository.delete(project.id)
|
||||
|
||||
logger.info(f"Project '{name}' removed from configuration and database")
|
||||
|
||||
|
||||
@@ -256,16 +256,24 @@ class SyncService:
|
||||
del self._file_failures[path]
|
||||
|
||||
@logfire.instrument()
|
||||
async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport:
|
||||
"""Sync all files with database and update scan watermark."""
|
||||
async def sync(
|
||||
self, directory: Path, project_name: Optional[str] = None, force_full: bool = False
|
||||
) -> SyncReport:
|
||||
"""Sync all files with database and update scan watermark.
|
||||
|
||||
Args:
|
||||
directory: Directory to sync
|
||||
project_name: Optional project name
|
||||
force_full: If True, force a full scan bypassing watermark optimization
|
||||
"""
|
||||
|
||||
start_time = time.time()
|
||||
sync_start_timestamp = time.time() # Capture at start for watermark
|
||||
logger.info(f"Sync operation started for directory: {directory}")
|
||||
logger.info(f"Sync operation started for directory: {directory} (force_full={force_full})")
|
||||
|
||||
# initial paths from db to sync
|
||||
# path -> checksum
|
||||
report = await self.scan(directory)
|
||||
report = await self.scan(directory, force_full=force_full)
|
||||
|
||||
# order of sync matters to resolve relations effectively
|
||||
logger.info(
|
||||
@@ -383,7 +391,7 @@ class SyncService:
|
||||
return report
|
||||
|
||||
@logfire.instrument()
|
||||
async def scan(self, directory):
|
||||
async def scan(self, directory, force_full: bool = False):
|
||||
"""Smart scan using watermark and file count for large project optimization.
|
||||
|
||||
Uses scan watermark tracking to dramatically reduce scan time for large projects:
|
||||
@@ -401,6 +409,10 @@ class SyncService:
|
||||
- Compare with last_file_count to detect deletions
|
||||
- If no deletions: incremental scan with find -newermt (0.2s)
|
||||
- Process changed files with mtime-based comparison
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
force_full: If True, bypass watermark optimization and force full scan
|
||||
"""
|
||||
scan_start_time = time.time()
|
||||
|
||||
@@ -420,7 +432,13 @@ class SyncService:
|
||||
logger.debug(f"Found {current_count} files in directory")
|
||||
|
||||
# Step 2: Determine scan strategy based on watermark and file count
|
||||
if project.last_file_count is None:
|
||||
if force_full:
|
||||
# User explicitly requested full scan → bypass watermark optimization
|
||||
scan_type = "full_forced"
|
||||
logger.info("Force full scan requested, bypassing watermark optimization")
|
||||
file_paths_to_scan = await self._scan_directory_full(directory)
|
||||
|
||||
elif project.last_file_count is None:
|
||||
# First sync ever → full scan
|
||||
scan_type = "full_initial"
|
||||
logger.info("First sync for this project, performing full scan")
|
||||
@@ -550,7 +568,7 @@ class SyncService:
|
||||
|
||||
# Step 5: Detect deletions (only for full scans)
|
||||
# Incremental scans can't reliably detect deletions since they only see modified files
|
||||
if scan_type in ("full_initial", "full_deletions", "full_fallback"):
|
||||
if scan_type in ("full_initial", "full_deletions", "full_fallback", "full_forced"):
|
||||
# Use optimized query for just file paths (not full entities)
|
||||
db_file_paths = await self.entity_repository.get_all_file_paths()
|
||||
logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection")
|
||||
|
||||
@@ -13,6 +13,49 @@ from loguru import logger
|
||||
from unidecode import unidecode
|
||||
|
||||
|
||||
def normalize_project_path(path: str) -> str:
|
||||
"""Normalize project path by stripping mount point prefix.
|
||||
|
||||
In cloud deployments, the S3 bucket is mounted at /app/data. We strip this
|
||||
prefix from project paths to avoid leaking implementation details and to
|
||||
ensure paths match the actual S3 bucket structure.
|
||||
|
||||
For local paths (including Windows paths), returns the path unchanged.
|
||||
|
||||
Args:
|
||||
path: Project path (e.g., "/app/data/basic-memory-llc" or "C:\\Users\\...")
|
||||
|
||||
Returns:
|
||||
Normalized path (e.g., "/basic-memory-llc" or "C:\\Users\\...")
|
||||
|
||||
Examples:
|
||||
>>> normalize_project_path("/app/data/my-project")
|
||||
'/my-project'
|
||||
>>> normalize_project_path("/my-project")
|
||||
'/my-project'
|
||||
>>> normalize_project_path("app/data/my-project")
|
||||
'/my-project'
|
||||
>>> normalize_project_path("C:\\\\Users\\\\project")
|
||||
'C:\\\\Users\\\\project'
|
||||
"""
|
||||
# Check if this is a Windows absolute path (e.g., C:\Users\...)
|
||||
# Windows paths have a drive letter followed by a colon
|
||||
if len(path) >= 2 and path[1] == ":":
|
||||
# Windows absolute path - return unchanged
|
||||
return path
|
||||
|
||||
# Handle both absolute and relative Unix paths
|
||||
normalized = path.lstrip("/")
|
||||
if normalized.startswith("app/data/"):
|
||||
normalized = normalized.removeprefix("app/data/")
|
||||
|
||||
# Ensure leading slash for Unix absolute paths
|
||||
if not normalized.startswith("/"):
|
||||
normalized = "/" + normalized
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PathLike(Protocol):
|
||||
"""Protocol for objects that can be used as paths."""
|
||||
|
||||
Reference in New Issue
Block a user