admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: SPEC-20 enhancements - cleanup, path normalization, and docs

Browse Source 
This commit adds several critical improvements discovered during
manual testing of SPEC-20 project-scoped rclone sync:

**Critical Bug Fixes:**

1. Path Normalization (fixes path doubling bug)
   - API: Strip /app/data/ prefix in project_router.py
   - CLI: Defensive normalization in project.py
   - Rclone: Fix get_project_remote() path construction
   - Prevents files syncing to /app/data/app/data/project/

2. Rclone Flag Fix
   - Changed --filters-file to correct --filter-from flag
   - Fixes "unknown flag" error in sync and bisync

**Enhancements:**

3. Automatic Database Sync
   - POST to /{project}/project/sync after file operations
   - Keeps database in sync with files automatically
   - Skipped on --dry-run operations

4. Enhanced Project Removal
   - Clean up local sync directory (with --delete-notes)
   - Always remove bisync state directory
   - Always remove cloud_projects config entry
   - Informative messages about what was/wasn't deleted

5. Bisync State Reset Command
   - New: bm project bisync-reset <project>
   - Clears corrupted bisync metadata
   - Safe recovery tool for bisync issues

6. Improved Project List UI
   - Show Local Path column in cloud mode
   - Conditionally show/hide columns based on config
   - Prevent path truncation with no_wrap/overflow
   - Apply path normalization to display

**Documentation:**

7. Cloud CLI Documentation
   - Add troubleshooting: empty directory bisync issues
   - Add troubleshooting: bisync state corruption
   - Document bisync-reset command usage
   - Explain rclone bisync limitations

8. SPEC-20 Updates
   - Mark implementation complete
   - Document all enhancements in Implementation Notes
   - Update phase checklists with completed work
   - Add manual testing results

**Tests:**

9. Unit Tests for --local-path
   - Test config persistence with --local-path
   - Test no config without --local-path
   - Test tilde expansion
   - Test nested directory creation

All changes tested manually end-to-end.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
phernandez
2025-10-28 17:36:36 -05:00
parent db85186e37
commit d749c7737f
7 changed files with 630 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/api/routers/project_router.py
+20 -2
View File
@@ -27,6 +27,24 @@ project_router = APIRouter(prefix="/project", tags=["project"])
project_resource_router = APIRouter(prefix="/projects", tags=["project_management"])
def normalize_project_path(path: str) -> str:
"""Normalize project path by stripping mount point prefix.
In cloud deployments, the S3 bucket is mounted at /app/data. We strip this
prefix from project paths to avoid leaking implementation details and to
ensure paths match the actual S3 bucket structure.
Args:
path: Project path (e.g., "/app/data/basic-memory-llc")
Returns:
Normalized path (e.g., "/basic-memory-llc")
"""
if path.startswith("/app/data/"):
return path.removeprefix("/app/data")
return path
@project_router.get("/info", response_model=ProjectInfoResponse)
async def get_project_info(
project_service: ProjectServiceDep,
@@ -50,7 +68,7 @@ async def get_project(
return ProjectItem(
name=found_project.name,
path=found_project.path,
path=normalize_project_path(found_project.path),
is_default=found_project.is_default or False,
)
@@ -167,7 +185,7 @@ async def list_projects(
project_items = [
ProjectItem(
name=project.name,
path=project.path,
path=normalize_project_path(project.path),
is_default=project.is_default or False,
)
for project in projects
src/basic_memory/cli/commands/cloud/core_commands.py
+12 -5
View File
@@ -174,11 +174,18 @@ def setup() -> None:
console.print("\n[bold green]✓ Cloud setup completed successfully![/bold green]")
console.print("\n[bold]Next steps:[/bold]")
console.print("1. Add a project with local sync path:")
console.print(" bm project add research ~/projects/research --local-path ~/sync/research")
console.print("\n2. Sync your project:")
console.print(" bm project bisync --name research --resync # First time")
console.print(" bm project bisync --name research # Subsequent syncs")
console.print("\n[dim]Use 'bm project --help' for more commands[/dim]")
console.print(" bm project add research --local-path ~/Documents/research")
console.print("\n Or configure sync for an existing project:")
console.print(" bm project sync-setup research ~/Documents/research")
console.print("\n2. Preview the initial sync (recommended):")
console.print(" bm project bisync --name research --resync --dry-run")
console.print("\n3. If all looks good, run the actual sync:")
console.print(" bm project bisync --name research --resync")
console.print("\n4. Subsequent syncs (no --resync needed):")
console.print(" bm project bisync --name research")
console.print(
"\n[dim]Tip: Always use --dry-run first to preview changes before syncing[/dim]"
)
except (RcloneInstallError, BisyncError, CloudAPIError) as e:
console.print(f"\n[red]Setup failed: {e}[/red]")
src/basic_memory/cli/commands/cloud/rclone_commands.py
+11 -4
View File
@@ -90,10 +90,17 @@ def get_project_remote(project: SyncProject, bucket_name: str) -> str:
bucket_name: S3 bucket name
Returns:
Remote path like "basic-memory-cloud:bucket-name/app/data/research"
Remote path like "basic-memory-cloud:bucket-name/basic-memory-llc"
Note:
The API returns paths like "/app/data/basic-memory-llc" because the S3 bucket
is mounted at /app/data on the fly machine. We need to strip the /app/data/
prefix to get the actual S3 path within the bucket.
"""
# Strip leading slash from cloud path
# Strip /app/data/ prefix from cloud path (mount point on fly machine)
cloud_path = project.path.lstrip("/")
if cloud_path.startswith("app/data/"):
cloud_path = cloud_path.removeprefix("app/data/")
return f"basic-memory-cloud:{bucket_name}/{cloud_path}"
@@ -131,7 +138,7 @@ def project_sync(
"sync",
str(local_path),
remote_path,
"--filters-file",
"--filter-from",
str(filter_path),
]
@@ -194,7 +201,7 @@ def project_bisync(
"--resilient",
"--conflict-resolve=newer",
"--max-delete=25",
"--filters-file",
"--filter-from",
str(filter_path),
"--workdir",
str(state_path),
src/basic_memory/cli/commands/project.py
+223 -38
View File
@@ -43,6 +43,23 @@ project_app = typer.Typer(help="Manage multiple Basic Memory projects")
app.add_typer(project_app, name="project")
def normalize_project_path(path: str) -> str:
"""Normalize project path by stripping mount point prefix.
In cloud deployments, the S3 bucket is mounted at /app/data. We strip this
prefix to get the actual S3 bucket path and avoid leaking implementation details.
Args:
path: Project path (e.g., "/app/data/basic-memory-llc")
Returns:
Normalized path (e.g., "/basic-memory-llc")
"""
if path.startswith("/app/data/"):
return path.removeprefix("/app/data")
return path
def format_path(path: str) -> str:
"""Format a path for display, using ~ for home directory."""
home = str(Path.home())
@@ -62,15 +79,40 @@ def list_projects() -> None:
try:
result = asyncio.run(_list_projects())
config = ConfigManager().config
table = Table(title="Basic Memory Projects")
table.add_column("Name", style="cyan")
table.add_column("Path", style="green")
table.add_column("Default", style="magenta")
# Add Local Path column if in cloud mode
if config.cloud_mode_enabled:
table.add_column("Local Path", style="yellow", no_wrap=True, overflow="fold")
# Only show Default column if default_project_mode is enabled
if config.default_project_mode:
table.add_column("Default", style="magenta")
for project in result.projects:
is_default = "✓" if project.is_default else ""
table.add_row(project.name, format_path(project.path), is_default)
normalized_path = normalize_project_path(project.path)
# Build row based on mode
row = [project.name, format_path(normalized_path)]
# Add local path if in cloud mode
if config.cloud_mode_enabled:
local_path = ""
if project.name in config.cloud_projects:
local_path = config.cloud_projects[project.name].local_path or ""
local_path = format_path(local_path)
row.append(local_path)
# Add default indicator if default_project_mode is enabled
if config.default_project_mode:
row.append(is_default)
table.add_row(*row)
console.print(table)
except Exception as e:
@@ -91,20 +133,22 @@ def add_project(
) -> None:
"""Add a new project.
Cloud mode examples:
bm project add research # No local sync
bm project add research --local-path ~/docs # With local sync
Cloud mode examples:\n
bm project add research # No local sync\n
bm project add research --local-path ~/docs # With local sync\n
Local mode example:
bm project add research ~/Documents/research
Local mode example:\n
bm project add research ~/Documents/research
"""
config = ConfigManager().config
# Resolve local sync path early (needed for both cloud and local mode)
local_sync_path: str | None = None
if local_path:
local_sync_path = Path(os.path.abspath(os.path.expanduser(local_path))).as_posix()
if config.cloud_mode_enabled:
# Cloud mode: path auto-generated from name, local sync is optional
local_sync_path = None
if local_path:
local_sync_path = Path(os.path.abspath(os.path.expanduser(local_path))).as_posix()
async def _add_project():
async with get_client() as client:
@@ -135,10 +179,26 @@ def add_project(
result = asyncio.run(_add_project())
console.print(f"[green]{result.message}[/green]")
# Show sync setup hint if in cloud mode and local sync configured
if config.cloud_mode_enabled and local_path:
console.print("\nTo sync this project:")
console.print(f" bm project bisync --name {name} --resync")
# Save local sync path to config if in cloud mode
if config.cloud_mode_enabled and local_sync_path:
from basic_memory.config import CloudProjectConfig
# Create local directory if it doesn't exist
local_dir = Path(local_sync_path)
local_dir.mkdir(parents=True, exist_ok=True)
# Update config with sync path
config.cloud_projects[name] = CloudProjectConfig(
local_path=local_sync_path,
last_sync=None,
bisync_initialized=False,
)
ConfigManager().save_config(config)
console.print(f"\n[green]✓ Local sync path configured: {local_sync_path}[/green]")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
except Exception as e:
console.print(f"[red]Error adding project: {str(e)}[/red]")
raise typer.Exit(1)
@@ -154,30 +214,46 @@ def setup_project_sync(
Example:
bm project sync-setup research ~/Documents/research
"""
config = ConfigManager().config
config_manager = ConfigManager()
config = config_manager.config
if not config.cloud_mode_enabled:
console.print("[red]Error: sync-setup only available in cloud mode[/red]")
raise typer.Exit(1)
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path))).as_posix()
async def _update_project():
async def _verify_project_exists():
"""Verify the project exists on cloud by listing all projects."""
async with get_client() as client:
data = {"local_sync_path": resolved_path}
project_permalink = generate_permalink(name)
response = await call_patch(
client,
f"/projects/{project_permalink}",
json=data,
)
return ProjectStatusResponse.model_validate(response.json())
response = await call_get(client, "/projects/projects")
project_list = response.json()
project_names = [p["name"] for p in project_list["projects"]]
if name not in project_names:
raise ValueError(f"Project '{name}' not found on cloud")
return True
try:
result = asyncio.run(_update_project())
console.print(f"[green]{result.message}[/green]")
console.print(f"\nLocal sync configured: {resolved_path}")
console.print(f"\nTo sync: bm project bisync --name {name} --resync")
# Verify project exists on cloud
asyncio.run(_verify_project_exists())
# Resolve and create local path
resolved_path = Path(os.path.abspath(os.path.expanduser(local_path)))
resolved_path.mkdir(parents=True, exist_ok=True)
# Update local config with sync path
from basic_memory.config import CloudProjectConfig
config.cloud_projects[name] = CloudProjectConfig(
local_path=resolved_path.as_posix(),
last_sync=None,
bisync_initialized=False,
)
config_manager.save_config(config)
console.print(f"[green]✓ Sync configured for project '{name}'[/green]")
console.print(f"\nLocal sync path: {resolved_path}")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
except Exception as e:
console.print(f"[red]Error configuring sync: {str(e)}[/red]")
raise typer.Exit(1)
@@ -201,16 +277,59 @@ def remove_project(
return ProjectStatusResponse.model_validate(response.json())
try:
# Get config to check for local sync path and bisync state
config = ConfigManager().config
local_path = None
has_bisync_state = False
if config.cloud_mode_enabled and name in config.cloud_projects:
local_path = config.cloud_projects[name].local_path
# Check for bisync state
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
bisync_state_path = get_project_bisync_state(name)
has_bisync_state = bisync_state_path.exists()
# Remove project from cloud/API
result = asyncio.run(_remove_project())
console.print(f"[green]{result.message}[/green]")
# Clean up local sync directory if it exists and delete_notes is True
if delete_notes and local_path:
local_dir = Path(local_path)
if local_dir.exists():
import shutil
shutil.rmtree(local_dir)
console.print(f"[green]✓ Removed local sync directory: {local_path}[/green]")
# Clean up bisync state if it exists
if has_bisync_state:
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
import shutil
bisync_state_path = get_project_bisync_state(name)
if bisync_state_path.exists():
shutil.rmtree(bisync_state_path)
console.print("[green]✓ Removed bisync state[/green]")
# Clean up cloud_projects config entry
if config.cloud_mode_enabled and name in config.cloud_projects:
del config.cloud_projects[name]
ConfigManager().save_config(config)
# Show informative message if files were not deleted
if not delete_notes:
if local_path:
console.print(f"[yellow]Note: Local files remain at {local_path}[/yellow]")
else:
console.print("[yellow]Note: Cloud project files have not been deleted.[/yellow]")
except Exception as e:
console.print(f"[red]Error removing project: {str(e)}[/red]")
raise typer.Exit(1)
# Show this message only if files were not deleted
if not delete_notes:
console.print("[yellow]Note: The project files have not been deleted from disk.[/yellow]")
@project_app.command("default")
def set_default_project(
@@ -365,7 +484,7 @@ def sync_project_command(
# Create SyncProject
sync_project = SyncProject(
name=project_data.name,
path=project_data.path,
path=normalize_project_path(project_data.path),
local_sync_path=local_sync_path,
)
@@ -375,6 +494,21 @@ def sync_project_command(
if success:
console.print(f"[green]✓ {name} synced successfully[/green]")
# Trigger database sync if not a dry run
if not dry_run:
async def _trigger_db_sync():
async with get_client() as client:
permalink = generate_permalink(name)
response = await call_post(client, f"/{permalink}/project/sync", json={})
return response.json()
try:
result = asyncio.run(_trigger_db_sync())
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
except Exception as e:
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
else:
console.print(f"[red]✗ {name} sync failed[/red]")
raise typer.Exit(1)
@@ -439,7 +573,7 @@ def bisync_project_command(
# Create SyncProject
sync_project = SyncProject(
name=project_data.name,
path=project_data.path,
path=normalize_project_path(project_data.path),
local_sync_path=local_sync_path,
)
@@ -451,6 +585,26 @@ def bisync_project_command(
if success:
console.print(f"[green]✓ {name} bisync completed successfully[/green]")
# Update config
config.cloud_projects[name].last_sync = datetime.now()
config.cloud_projects[name].bisync_initialized = True
ConfigManager().save_config(config)
# Trigger database sync if not a dry run
if not dry_run:
async def _trigger_db_sync():
async with get_client() as client:
permalink = generate_permalink(name)
response = await call_post(client, f"/{permalink}/project/sync", json={})
return response.json()
try:
result = asyncio.run(_trigger_db_sync())
console.print(f"[dim]Database sync initiated: {result.get('message')}[/dim]")
except Exception as e:
console.print(f"[yellow]Warning: Could not trigger database sync: {e}[/yellow]")
else:
console.print(f"[red]✗ {name} bisync failed[/red]")
raise typer.Exit(1)
@@ -511,7 +665,7 @@ def check_project_command(
# Create SyncProject
sync_project = SyncProject(
name=project_data.name,
path=project_data.path,
path=normalize_project_path(project_data.path),
local_sync_path=local_sync_path,
)
@@ -532,6 +686,37 @@ def check_project_command(
raise typer.Exit(1)
@project_app.command("bisync-reset")
def bisync_reset(
name: str = typer.Argument(..., help="Project name to reset bisync state for"),
) -> None:
"""Clear bisync state for a project.
This removes the bisync metadata files, forcing a fresh --resync on next bisync.
Useful when bisync gets into an inconsistent state or when remote path changes.
"""
from basic_memory.cli.commands.cloud.rclone_commands import get_project_bisync_state
import shutil
try:
state_path = get_project_bisync_state(name)
if not state_path.exists():
console.print(f"[yellow]No bisync state found for project '{name}'[/yellow]")
return
# Remove the entire state directory
shutil.rmtree(state_path)
console.print(f"[green]✓ Cleared bisync state for project '{name}'[/green]")
console.print("\nNext steps:")
console.print(f" 1. Preview: bm project bisync --name {name} --resync --dry-run")
console.print(f" 2. Sync: bm project bisync --name {name} --resync")
except Exception as e:
console.print(f"[red]Error clearing bisync state: {str(e)}[/red]")
raise typer.Exit(1)
@project_app.command("ls")
def ls_project_command(
name: str = typer.Option(..., "--name", help="Project name to list files from"),
@@ -571,7 +756,7 @@ def ls_project_command(
# Create SyncProject (local_sync_path not needed for ls)
sync_project = SyncProject(
name=project_data.name,
path=project_data.path,
path=normalize_project_path(project_data.path),
)
# List files