feat(rclone): Add simplified configure_rclone_remote() function (SPEC-20 Phase 2)

Add new configure_rclone_remote() that uses single remote name:
- Single remote: "basic-memory-cloud" (not tenant-specific)
- Simplifies from per-tenant remotes to one credential set per user
- Maintains backup_rclone_config() for safety
- Add comprehensive functional tests for rclone config
- Test remote configuration, updates, save/load operations

Old add_tenant_to_rclone_config() kept for backward compatibility
but will be removed in Phase 5.

Part of SPEC-20 Simplified Project-Scoped Rclone Sync.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
phernandez
2025-10-28 11:02:38 -05:00
parent 8bc550f613
commit 7c2b8b5a58
7 changed files with 193 additions and 25 deletions
@@ -985,11 +985,11 @@ rm -rf ~/basic-memory-cloud-sync/
## Implementation Checklist
### Phase 1: Config Schema (1-2 days)
- [ ] Add `CloudProjectConfig` model to `basic_memory/config.py`
- [ ] Add `cloud_projects: dict[str, CloudProjectConfig]` to Config model
- [ ] Test config loading/saving with new schema
- [ ] Handle migration from old config format
### Phase 1: Config Schema (1-2 days)
- [x] Add `CloudProjectConfig` model to `basic_memory/config.py`
- [x] Add `cloud_projects: dict[str, CloudProjectConfig]` to Config model
- [x] Test config loading/saving with new schema
- [x] Handle migration from old config format
### Phase 2: Rclone Config Simplification (1 day)
- [ ] Update `configure_rclone_remote()` to use `basic-memory-cloud` as remote name
@@ -116,6 +116,53 @@ def save_rclone_config(config: configparser.ConfigParser) -> None:
console.print(f"[dim]Updated rclone config: {config_path}[/dim]")
def configure_rclone_remote(
access_key: str,
secret_key: str,
endpoint: str = "https://fly.storage.tigris.dev",
region: str = "auto",
) -> str:
"""Configure single rclone remote named 'basic-memory-cloud'.
This is the simplified approach from SPEC-20 that uses one remote
for all Basic Memory cloud operations (not tenant-specific).
Args:
access_key: S3 access key ID
secret_key: S3 secret access key
endpoint: S3-compatible endpoint URL
region: S3 region (default: auto)
Returns:
The remote name: "basic-memory-cloud"
"""
# Backup existing config
backup_rclone_config()
# Load existing config
config = load_rclone_config()
# Single remote name (not tenant-specific)
REMOTE_NAME = "basic-memory-cloud"
# Add/update the remote section
if not config.has_section(REMOTE_NAME):
config.add_section(REMOTE_NAME)
config.set(REMOTE_NAME, "type", "s3")
config.set(REMOTE_NAME, "provider", "Other")
config.set(REMOTE_NAME, "access_key_id", access_key)
config.set(REMOTE_NAME, "secret_access_key", secret_key)
config.set(REMOTE_NAME, "endpoint", endpoint)
config.set(REMOTE_NAME, "region", region)
# Save updated config
save_rclone_config(config)
console.print(f"[green]✓ Configured rclone remote: {REMOTE_NAME}[/green]")
return REMOTE_NAME
def add_tenant_to_rclone_config(
tenant_id: str,
bucket_name: str,
+4 -8
View File
@@ -47,16 +47,12 @@ class CloudProjectConfig(BaseModel):
that is synced with Basic Memory Cloud.
"""
local_path: str = Field(
description="Local working directory path for this cloud project"
)
local_path: str = Field(description="Local working directory path for this cloud project")
last_sync: Optional[datetime] = Field(
default=None,
description="Timestamp of last successful sync operation"
default=None, description="Timestamp of last successful sync operation"
)
bisync_initialized: bool = Field(
default=False,
description="Whether rclone bisync baseline has been established"
default=False, description="Whether rclone bisync baseline has been established"
)
@@ -454,7 +450,7 @@ def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None
"""Save configuration to file."""
try:
# Use model_dump with mode='json' to serialize datetime objects properly
config_dict = config.model_dump(mode='json')
config_dict = config.model_dump(mode="json")
file_path.write_text(json.dumps(config_dict, indent=2))
except Exception as e: # pragma: no cover
logger.error(f"Failed to save config: {e}")
@@ -6,7 +6,6 @@ but later code expects strings and calls .strip() on them, causing AttributeErro
"""
import pytest
from pathlib import Path
from basic_memory.markdown.entity_parser import EntityParser
@@ -225,11 +224,13 @@ This file has datetime values in frontmatter that PyYAML will parse as datetime
created_at = entity_markdown.frontmatter.metadata.get("created_at")
assert isinstance(created_at, str), "Datetime should be converted to string"
# PyYAML parses "2025-10-24 14:30:00" as datetime, which we normalize to ISO
assert "2025-10-24" in created_at and "14:30:00" in created_at, \
assert "2025-10-24" in created_at and "14:30:00" in created_at, (
f"Datetime with time should be normalized to ISO format, got: {created_at}"
)
updated_at = entity_markdown.frontmatter.metadata.get("updated_at")
assert isinstance(updated_at, str), "Datetime should be converted to string"
# PyYAML parses "2025-10-24T00:00:00" as datetime, which we normalize to ISO
assert "2025-10-24" in updated_at and "00:00:00" in updated_at, \
f"Datetime at midnight should be normalized to ISO format, got: {updated_at}"
assert "2025-10-24" in updated_at and "00:00:00" in updated_at, (
f"Datetime at midnight should be normalized to ISO format, got: {updated_at}"
)
+2
View File
@@ -1703,9 +1703,11 @@ Content 3
with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file):
# Fail 3 times for file1 and file2 (file3 succeeds each time)
await force_full_scan(sync_service)
await sync_service.sync(project_dir) # Fail count: file1=1, file2=1
await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan
await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan
await force_full_scan(sync_service)
await sync_service.sync(project_dir) # Fail count: file1=2, file2=2
await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan
await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan
+11 -8
View File
@@ -296,16 +296,18 @@ class TestConfigManager:
"research": CloudProjectConfig(
local_path=str(temp_path / "research-local"),
last_sync=now,
bisync_initialized=True
bisync_initialized=True,
)
}
},
)
config_manager.save_config(test_config)
# Load and verify
loaded_config = config_manager.load_config()
assert "research" in loaded_config.cloud_projects
assert loaded_config.cloud_projects["research"].local_path == str(temp_path / "research-local")
assert loaded_config.cloud_projects["research"].local_path == str(
temp_path / "research-local"
)
assert loaded_config.cloud_projects["research"].bisync_initialized is True
assert loaded_config.cloud_projects["research"].last_sync == now
@@ -320,9 +322,7 @@ class TestConfigManager:
config_manager.config_dir.mkdir(parents=True, exist_ok=True)
# Create initial config without cloud projects
initial_config = BasicMemoryConfig(
projects={"main": str(temp_path / "main")}
)
initial_config = BasicMemoryConfig(projects={"main": str(temp_path / "main")})
config_manager.save_config(initial_config)
# Load, modify, and save
@@ -337,7 +337,9 @@ class TestConfigManager:
# Reload and verify persistence
reloaded_config = config_manager.load_config()
assert "work" in reloaded_config.cloud_projects
assert reloaded_config.cloud_projects["work"].local_path == str(temp_path / "work-local")
assert reloaded_config.cloud_projects["work"].local_path == str(
temp_path / "work-local"
)
assert reloaded_config.cloud_projects["work"].bisync_initialized is False
def test_backward_compatibility_loading_config_without_cloud_projects(self):
@@ -352,11 +354,12 @@ class TestConfigManager:
# Manually write old-style config without cloud_projects
import json
old_config_data = {
"env": "dev",
"projects": {"main": str(temp_path / "main")},
"default_project": "main",
"log_level": "INFO"
"log_level": "INFO",
}
config_manager.config_file.write_text(json.dumps(old_config_data, indent=2))
+119
View File
@@ -0,0 +1,119 @@
"""Test rclone configuration management."""
import configparser
import tempfile
from pathlib import Path
import pytest
from basic_memory.cli.commands.cloud.rclone_config import (
configure_rclone_remote,
load_rclone_config,
save_rclone_config,
)
@pytest.fixture
def temp_rclone_config(monkeypatch):
"""Create a temporary rclone config directory."""
with tempfile.TemporaryDirectory() as temp_dir:
config_dir = Path(temp_dir) / ".config" / "rclone"
config_dir.mkdir(parents=True, exist_ok=True)
config_path = config_dir / "rclone.conf"
# Monkeypatch get_rclone_config_path to use temp directory
monkeypatch.setattr(
"basic_memory.cli.commands.cloud.rclone_config.get_rclone_config_path",
lambda: config_path,
)
yield config_path
def test_configure_rclone_remote(temp_rclone_config):
"""Test configuring simplified rclone remote."""
# Configure remote
remote_name = configure_rclone_remote(
access_key="test_access_key",
secret_key="test_secret_key",
endpoint="https://test.endpoint.com",
region="test-region",
)
# Should return correct remote name
assert remote_name == "basic-memory-cloud"
# Load and verify config
config = load_rclone_config()
# Should have the remote section
assert config.has_section("basic-memory-cloud")
# Should have correct settings
assert config.get("basic-memory-cloud", "type") == "s3"
assert config.get("basic-memory-cloud", "provider") == "Other"
assert config.get("basic-memory-cloud", "access_key_id") == "test_access_key"
assert config.get("basic-memory-cloud", "secret_access_key") == "test_secret_key"
assert config.get("basic-memory-cloud", "endpoint") == "https://test.endpoint.com"
assert config.get("basic-memory-cloud", "region") == "test-region"
def test_configure_rclone_remote_default_values(temp_rclone_config):
"""Test configuring remote with default endpoint and region."""
remote_name = configure_rclone_remote(access_key="test_key", secret_key="test_secret")
assert remote_name == "basic-memory-cloud"
config = load_rclone_config()
# Should use default values
assert config.get("basic-memory-cloud", "endpoint") == "https://fly.storage.tigris.dev"
assert config.get("basic-memory-cloud", "region") == "auto"
def test_configure_rclone_remote_updates_existing(temp_rclone_config):
"""Test that configuring remote updates existing configuration."""
# First configuration
configure_rclone_remote(access_key="old_key", secret_key="old_secret")
# Update configuration
configure_rclone_remote(access_key="new_key", secret_key="new_secret")
config = load_rclone_config()
# Should have updated values
assert config.get("basic-memory-cloud", "access_key_id") == "new_key"
assert config.get("basic-memory-cloud", "secret_access_key") == "new_secret"
# Should only have one section (not multiple)
sections = config.sections()
assert sections.count("basic-memory-cloud") == 1
def test_save_and_load_rclone_config(temp_rclone_config):
"""Test saving and loading rclone config."""
# Create config
config = configparser.ConfigParser()
config.add_section("test-remote")
config.set("test-remote", "type", "s3")
config.set("test-remote", "provider", "AWS")
# Save config
save_rclone_config(config)
# Load and verify
loaded_config = load_rclone_config()
assert loaded_config.has_section("test-remote")
assert loaded_config.get("test-remote", "type") == "s3"
assert loaded_config.get("test-remote", "provider") == "AWS"
def test_load_rclone_config_nonexistent(temp_rclone_config):
"""Test loading config when file doesn't exist."""
# Delete the config file if it exists
if temp_rclone_config.exists():
temp_rclone_config.unlink()
# Should return empty config without error
config = load_rclone_config()
assert len(config.sections()) == 0