admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add WebDAV upload command for cloud projects (#356)

Browse Source 
Signed-off-by: phernandez <paul@basicmachines.co>
Signed-off-by: Pablo Hernandez <pablo@basicmachines.co>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
2025-10-13 23:20:54 -05:00
committed by GitHub
parent e773c002ce
commit 5258f45730
11 changed files with 1050 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/cli/commands/cloud/__init__.py
+2 -1
View File
@@ -2,4 +2,5 @@
# Import all commands to register them with typer
from basic_memory.cli.commands.cloud.core_commands import * # noqa: F401,F403
from basic_memory.cli.commands.cloud.api_client import get_authenticated_headers # noqa: F401
from basic_memory.cli.commands.cloud.api_client import get_authenticated_headers, get_cloud_config # noqa: F401
from basic_memory.cli.commands.cloud.upload_command import * # noqa: F401,F403
src/basic_memory/cli/commands/cloud/bisync_commands.py
+4 -57
View File
@@ -12,6 +12,10 @@ from rich.console import Console
from rich.table import Table
from basic_memory.cli.commands.cloud.api_client import CloudAPIError, make_api_request
from basic_memory.cli.commands.cloud.cloud_utils import (
create_cloud_project,
fetch_cloud_projects,
)
from basic_memory.cli.commands.cloud.rclone_config import (
add_tenant_to_rclone_config,
)
@@ -21,11 +25,7 @@ from basic_memory.ignore_utils import get_bmignore_path, create_default_bmignore
from basic_memory.schemas.cloud import (
TenantMountInfo,
MountCredentials,
CloudProjectList,
CloudProjectCreateRequest,
CloudProjectCreateResponse,
)
from basic_memory.utils import generate_permalink
console = Console()
@@ -110,24 +110,6 @@ async def generate_mount_credentials(tenant_id: str) -> MountCredentials:
raise BisyncError(f"Failed to generate credentials: {e}") from e
async def fetch_cloud_projects() -> CloudProjectList:
"""Fetch list of projects from cloud API.
Returns:
CloudProjectList with projects from cloud
"""
try:
config_manager = ConfigManager()
config = config_manager.config
host_url = config.cloud_host.rstrip("/")
response = await make_api_request(method="GET", url=f"{host_url}/proxy/projects/projects")
return CloudProjectList.model_validate(response.json())
except Exception as e:
raise BisyncError(f"Failed to fetch cloud projects: {e}") from e
def scan_local_directories(sync_dir: Path) -> list[str]:
"""Scan local sync directory for project folders.
@@ -148,41 +130,6 @@ def scan_local_directories(sync_dir: Path) -> list[str]:
return directories
async def create_cloud_project(project_name: str) -> CloudProjectCreateResponse:
"""Create a new project on cloud.
Args:
project_name: Name of project to create
Returns:
CloudProjectCreateResponse with project details from API
"""
try:
config_manager = ConfigManager()
config = config_manager.config
host_url = config.cloud_host.rstrip("/")
# Use generate_permalink to ensure consistent naming
project_path = generate_permalink(project_name)
project_data = CloudProjectCreateRequest(
name=project_name,
path=project_path,
set_default=False,
)
response = await make_api_request(
method="POST",
url=f"{host_url}/proxy/projects/projects",
headers={"Content-Type": "application/json"},
json_data=project_data.model_dump(),
)
return CloudProjectCreateResponse.model_validate(response.json())
except Exception as e:
raise BisyncError(f"Failed to create cloud project '{project_name}': {e}") from e
def get_bisync_state_path(tenant_id: str) -> Path:
"""Get path to bisync state directory."""
return Path.home() / ".basic-memory" / "bisync-state" / tenant_id
src/basic_memory/cli/commands/cloud/cloud_utils.py
+100
View File
@@ -0,0 +1,100 @@
"""Shared utilities for cloud operations."""
from basic_memory.cli.commands.cloud.api_client import make_api_request
from basic_memory.config import ConfigManager
from basic_memory.schemas.cloud import (
CloudProjectList,
CloudProjectCreateRequest,
CloudProjectCreateResponse,
)
from basic_memory.utils import generate_permalink
class CloudUtilsError(Exception):
"""Exception raised for cloud utility errors."""
pass
async def fetch_cloud_projects() -> CloudProjectList:
"""Fetch list of projects from cloud API.
Returns:
CloudProjectList with projects from cloud
"""
try:
config_manager = ConfigManager()
config = config_manager.config
host_url = config.cloud_host.rstrip("/")
response = await make_api_request(method="GET", url=f"{host_url}/proxy/projects/projects")
return CloudProjectList.model_validate(response.json())
except Exception as e:
raise CloudUtilsError(f"Failed to fetch cloud projects: {e}") from e
async def create_cloud_project(project_name: str) -> CloudProjectCreateResponse:
"""Create a new project on cloud.
Args:
project_name: Name of project to create
Returns:
CloudProjectCreateResponse with project details from API
"""
try:
config_manager = ConfigManager()
config = config_manager.config
host_url = config.cloud_host.rstrip("/")
# Use generate_permalink to ensure consistent naming
project_path = generate_permalink(project_name)
project_data = CloudProjectCreateRequest(
name=project_name,
path=project_path,
set_default=False,
)
response = await make_api_request(
method="POST",
url=f"{host_url}/proxy/projects/projects",
headers={"Content-Type": "application/json"},
json_data=project_data.model_dump(),
)
return CloudProjectCreateResponse.model_validate(response.json())
except Exception as e:
raise CloudUtilsError(f"Failed to create cloud project '{project_name}': {e}") from e
async def sync_project(project_name: str) -> None:
"""Trigger sync for a specific project on cloud.
Args:
project_name: Name of project to sync
"""
try:
from basic_memory.cli.commands.command_utils import run_sync
await run_sync(project=project_name)
except Exception as e:
raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
async def project_exists(project_name: str) -> bool:
"""Check if a project exists on cloud.
Args:
project_name: Name of project to check
Returns:
True if project exists, False otherwise
"""
try:
projects = await fetch_cloud_projects()
project_names = {p.name for p in projects.projects}
return project_name in project_names
except Exception:
return False
src/basic_memory/cli/commands/cloud/upload.py
+128
View File
@@ -0,0 +1,128 @@
"""WebDAV upload functionality for basic-memory projects."""
import os
from pathlib import Path
import aiofiles
import httpx
from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
from basic_memory.mcp.async_client import get_client
from basic_memory.mcp.tools.utils import call_put
async def upload_path(local_path: Path, project_name: str) -> bool:
"""
Upload a file or directory to cloud project via WebDAV.
Args:
local_path: Path to local file or directory
project_name: Name of cloud project (destination)
Returns:
True if upload succeeded, False otherwise
"""
try:
# Resolve path
local_path = local_path.resolve()
# Check if path exists
if not local_path.exists():
print(f"Error: Path does not exist: {local_path}")
return False
# Get files to upload
if local_path.is_file():
files_to_upload = [(local_path, local_path.name)]
else:
files_to_upload = _get_files_to_upload(local_path)
if not files_to_upload:
print("No files found to upload")
return True
print(f"Found {len(files_to_upload)} file(s) to upload")
# Upload files using httpx
total_bytes = 0
async with get_client() as client:
for i, (file_path, relative_path) in enumerate(files_to_upload, 1):
# Build remote path: /webdav/{project_name}/{relative_path}
remote_path = f"/webdav/{project_name}/{relative_path}"
print(f"Uploading {relative_path} ({i}/{len(files_to_upload)})")
# Read file content asynchronously
async with aiofiles.open(file_path, "rb") as f:
content = await f.read()
# Upload via HTTP PUT to WebDAV endpoint
response = await call_put(client, remote_path, content=content)
response.raise_for_status()
total_bytes += file_path.stat().st_size
# Format size based on magnitude
if total_bytes < 1024:
size_str = f"{total_bytes} bytes"
elif total_bytes < 1024 * 1024:
size_str = f"{total_bytes / 1024:.1f} KB"
else:
size_str = f"{total_bytes / (1024 * 1024):.1f} MB"
print(f"✓ Upload complete: {len(files_to_upload)} file(s) ({size_str})")
return True
except httpx.HTTPStatusError as e:
print(f"Upload failed: HTTP {e.response.status_code} - {e.response.text}")
return False
except Exception as e:
print(f"Upload failed: {e}")
return False
def _get_files_to_upload(directory: Path) -> list[tuple[Path, str]]:
"""
Get list of files to upload from directory.
Uses .bmignore and .gitignore patterns for filtering.
Args:
directory: Directory to scan
Returns:
List of (absolute_path, relative_path) tuples
"""
files = []
# Load ignore patterns from .bmignore and .gitignore
ignore_patterns = load_gitignore_patterns(directory)
# Walk through directory
for root, dirs, filenames in os.walk(directory):
root_path = Path(root)
# Filter directories based on ignore patterns
filtered_dirs = []
for d in dirs:
dir_path = root_path / d
if not should_ignore_path(dir_path, directory, ignore_patterns):
filtered_dirs.append(d)
dirs[:] = filtered_dirs
# Process files
for filename in filenames:
file_path = root_path / filename
# Check if file should be ignored
if should_ignore_path(file_path, directory, ignore_patterns):
continue
# Calculate relative path for remote
rel_path = file_path.relative_to(directory)
# Use forward slashes for WebDAV paths
remote_path = str(rel_path).replace("\\", "/")
files.append((file_path, remote_path))
return files
src/basic_memory/cli/commands/cloud/upload_command.py
+94
View File
@@ -0,0 +1,94 @@
"""Upload CLI commands for basic-memory projects."""
import asyncio
from pathlib import Path
import typer
from rich.console import Console
from basic_memory.cli.app import cloud_app
from basic_memory.cli.commands.cloud.cloud_utils import (
create_cloud_project,
project_exists,
sync_project,
)
from basic_memory.cli.commands.cloud.upload import upload_path
console = Console()
@cloud_app.command("upload")
def upload(
path: Path = typer.Argument(
...,
help="Path to local file or directory to upload",
exists=True,
readable=True,
resolve_path=True,
),
project: str = typer.Option(
...,
"--project",
"-p",
help="Cloud project name (destination)",
),
create_project: bool = typer.Option(
False,
"--create-project",
"-c",
help="Create project if it doesn't exist",
),
sync: bool = typer.Option(
True,
"--sync/--no-sync",
help="Sync project after upload (default: true)",
),
) -> None:
"""Upload local files or directories to cloud project via WebDAV.
Examples:
bm cloud upload ~/my-notes --project research
bm cloud upload notes.md --project research --create-project
bm cloud upload ~/docs --project work --no-sync
"""
async def _upload():
# Check if project exists
if not await project_exists(project):
if create_project:
console.print(f"[blue]Creating cloud project '{project}'...[/blue]")
try:
await create_cloud_project(project)
console.print(f"[green]✓ Created project '{project}'[/green]")
except Exception as e:
console.print(f"[red]Failed to create project: {e}[/red]")
raise typer.Exit(1)
else:
console.print(
f"[red]Project '{project}' does not exist.[/red]\n"
f"[yellow]Options:[/yellow]\n"
f" 1. Create it first: bm project add {project}\n"
f" 2. Use --create-project flag to create automatically"
)
raise typer.Exit(1)
# Perform upload
console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]")
success = await upload_path(path, project)
if not success:
console.print("[red]Upload failed[/red]")
raise typer.Exit(1)
console.print(f"[green]✅ Successfully uploaded to '{project}'[/green]")
# Sync project if requested
if sync:
console.print(f"[blue]Syncing project '{project}'...[/blue]")
try:
await sync_project(project)
console.print("[green]✓ Project synced[/green]")
except Exception as e:
console.print(f"[yellow]Warning: Sync failed: {e}[/yellow]")
console.print("[dim]Files uploaded but may not be indexed yet[/dim]")
asyncio.run(_upload())
src/basic_memory/mcp/async_client.py
+1
View File
@@ -87,6 +87,7 @@ async def get_client() -> AsyncIterator[AsyncClient]:
# Auth header set ONCE at client creation
proxy_base_url = f"{config.cloud_host}/proxy"
logger.info(f"Creating HTTP client for cloud proxy at: {proxy_base_url}")
print(f"proxy_base_url: {proxy_base_url}")
async with AsyncClient(
base_url=proxy_base_url,
headers={"Authorization": f"Bearer {token}"},