mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
feat(rclone): Add simplified configure_rclone_remote() function (SPEC-20 Phase 2)
Add new configure_rclone_remote() that uses single remote name: - Single remote: "basic-memory-cloud" (not tenant-specific) - Simplifies from per-tenant remotes to one credential set per user - Maintains backup_rclone_config() for safety - Add comprehensive functional tests for rclone config - Test remote configuration, updates, save/load operations Old add_tenant_to_rclone_config() kept for backward compatibility but will be removed in Phase 5. Part of SPEC-20 Simplified Project-Scoped Rclone Sync. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
@@ -985,11 +985,11 @@ rm -rf ~/basic-memory-cloud-sync/
|
||||
|
||||
## Implementation Checklist
|
||||
|
||||
### Phase 1: Config Schema (1-2 days)
|
||||
- [ ] Add `CloudProjectConfig` model to `basic_memory/config.py`
|
||||
- [ ] Add `cloud_projects: dict[str, CloudProjectConfig]` to Config model
|
||||
- [ ] Test config loading/saving with new schema
|
||||
- [ ] Handle migration from old config format
|
||||
### Phase 1: Config Schema (1-2 days) ✅
|
||||
- [x] Add `CloudProjectConfig` model to `basic_memory/config.py`
|
||||
- [x] Add `cloud_projects: dict[str, CloudProjectConfig]` to Config model
|
||||
- [x] Test config loading/saving with new schema
|
||||
- [x] Handle migration from old config format
|
||||
|
||||
### Phase 2: Rclone Config Simplification (1 day)
|
||||
- [ ] Update `configure_rclone_remote()` to use `basic-memory-cloud` as remote name
|
||||
|
||||
@@ -116,6 +116,53 @@ def save_rclone_config(config: configparser.ConfigParser) -> None:
|
||||
console.print(f"[dim]Updated rclone config: {config_path}[/dim]")
|
||||
|
||||
|
||||
def configure_rclone_remote(
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
endpoint: str = "https://fly.storage.tigris.dev",
|
||||
region: str = "auto",
|
||||
) -> str:
|
||||
"""Configure single rclone remote named 'basic-memory-cloud'.
|
||||
|
||||
This is the simplified approach from SPEC-20 that uses one remote
|
||||
for all Basic Memory cloud operations (not tenant-specific).
|
||||
|
||||
Args:
|
||||
access_key: S3 access key ID
|
||||
secret_key: S3 secret access key
|
||||
endpoint: S3-compatible endpoint URL
|
||||
region: S3 region (default: auto)
|
||||
|
||||
Returns:
|
||||
The remote name: "basic-memory-cloud"
|
||||
"""
|
||||
# Backup existing config
|
||||
backup_rclone_config()
|
||||
|
||||
# Load existing config
|
||||
config = load_rclone_config()
|
||||
|
||||
# Single remote name (not tenant-specific)
|
||||
REMOTE_NAME = "basic-memory-cloud"
|
||||
|
||||
# Add/update the remote section
|
||||
if not config.has_section(REMOTE_NAME):
|
||||
config.add_section(REMOTE_NAME)
|
||||
|
||||
config.set(REMOTE_NAME, "type", "s3")
|
||||
config.set(REMOTE_NAME, "provider", "Other")
|
||||
config.set(REMOTE_NAME, "access_key_id", access_key)
|
||||
config.set(REMOTE_NAME, "secret_access_key", secret_key)
|
||||
config.set(REMOTE_NAME, "endpoint", endpoint)
|
||||
config.set(REMOTE_NAME, "region", region)
|
||||
|
||||
# Save updated config
|
||||
save_rclone_config(config)
|
||||
|
||||
console.print(f"[green]✓ Configured rclone remote: {REMOTE_NAME}[/green]")
|
||||
return REMOTE_NAME
|
||||
|
||||
|
||||
def add_tenant_to_rclone_config(
|
||||
tenant_id: str,
|
||||
bucket_name: str,
|
||||
|
||||
@@ -47,16 +47,12 @@ class CloudProjectConfig(BaseModel):
|
||||
that is synced with Basic Memory Cloud.
|
||||
"""
|
||||
|
||||
local_path: str = Field(
|
||||
description="Local working directory path for this cloud project"
|
||||
)
|
||||
local_path: str = Field(description="Local working directory path for this cloud project")
|
||||
last_sync: Optional[datetime] = Field(
|
||||
default=None,
|
||||
description="Timestamp of last successful sync operation"
|
||||
default=None, description="Timestamp of last successful sync operation"
|
||||
)
|
||||
bisync_initialized: bool = Field(
|
||||
default=False,
|
||||
description="Whether rclone bisync baseline has been established"
|
||||
default=False, description="Whether rclone bisync baseline has been established"
|
||||
)
|
||||
|
||||
|
||||
@@ -454,7 +450,7 @@ def save_basic_memory_config(file_path: Path, config: BasicMemoryConfig) -> None
|
||||
"""Save configuration to file."""
|
||||
try:
|
||||
# Use model_dump with mode='json' to serialize datetime objects properly
|
||||
config_dict = config.model_dump(mode='json')
|
||||
config_dict = config.model_dump(mode="json")
|
||||
file_path.write_text(json.dumps(config_dict, indent=2))
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to save config: {e}")
|
||||
|
||||
@@ -6,7 +6,6 @@ but later code expects strings and calls .strip() on them, causing AttributeErro
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
|
||||
|
||||
@@ -225,11 +224,13 @@ This file has datetime values in frontmatter that PyYAML will parse as datetime
|
||||
created_at = entity_markdown.frontmatter.metadata.get("created_at")
|
||||
assert isinstance(created_at, str), "Datetime should be converted to string"
|
||||
# PyYAML parses "2025-10-24 14:30:00" as datetime, which we normalize to ISO
|
||||
assert "2025-10-24" in created_at and "14:30:00" in created_at, \
|
||||
assert "2025-10-24" in created_at and "14:30:00" in created_at, (
|
||||
f"Datetime with time should be normalized to ISO format, got: {created_at}"
|
||||
)
|
||||
|
||||
updated_at = entity_markdown.frontmatter.metadata.get("updated_at")
|
||||
assert isinstance(updated_at, str), "Datetime should be converted to string"
|
||||
# PyYAML parses "2025-10-24T00:00:00" as datetime, which we normalize to ISO
|
||||
assert "2025-10-24" in updated_at and "00:00:00" in updated_at, \
|
||||
f"Datetime at midnight should be normalized to ISO format, got: {updated_at}"
|
||||
assert "2025-10-24" in updated_at and "00:00:00" in updated_at, (
|
||||
f"Datetime at midnight should be normalized to ISO format, got: {updated_at}"
|
||||
)
|
||||
|
||||
@@ -1703,9 +1703,11 @@ Content 3
|
||||
|
||||
with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file):
|
||||
# Fail 3 times for file1 and file2 (file3 succeeds each time)
|
||||
await force_full_scan(sync_service)
|
||||
await sync_service.sync(project_dir) # Fail count: file1=1, file2=1
|
||||
await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan
|
||||
await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan
|
||||
await force_full_scan(sync_service)
|
||||
await sync_service.sync(project_dir) # Fail count: file1=2, file2=2
|
||||
await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan
|
||||
await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan
|
||||
|
||||
+11
-8
@@ -296,16 +296,18 @@ class TestConfigManager:
|
||||
"research": CloudProjectConfig(
|
||||
local_path=str(temp_path / "research-local"),
|
||||
last_sync=now,
|
||||
bisync_initialized=True
|
||||
bisync_initialized=True,
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
config_manager.save_config(test_config)
|
||||
|
||||
# Load and verify
|
||||
loaded_config = config_manager.load_config()
|
||||
assert "research" in loaded_config.cloud_projects
|
||||
assert loaded_config.cloud_projects["research"].local_path == str(temp_path / "research-local")
|
||||
assert loaded_config.cloud_projects["research"].local_path == str(
|
||||
temp_path / "research-local"
|
||||
)
|
||||
assert loaded_config.cloud_projects["research"].bisync_initialized is True
|
||||
assert loaded_config.cloud_projects["research"].last_sync == now
|
||||
|
||||
@@ -320,9 +322,7 @@ class TestConfigManager:
|
||||
config_manager.config_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create initial config without cloud projects
|
||||
initial_config = BasicMemoryConfig(
|
||||
projects={"main": str(temp_path / "main")}
|
||||
)
|
||||
initial_config = BasicMemoryConfig(projects={"main": str(temp_path / "main")})
|
||||
config_manager.save_config(initial_config)
|
||||
|
||||
# Load, modify, and save
|
||||
@@ -337,7 +337,9 @@ class TestConfigManager:
|
||||
# Reload and verify persistence
|
||||
reloaded_config = config_manager.load_config()
|
||||
assert "work" in reloaded_config.cloud_projects
|
||||
assert reloaded_config.cloud_projects["work"].local_path == str(temp_path / "work-local")
|
||||
assert reloaded_config.cloud_projects["work"].local_path == str(
|
||||
temp_path / "work-local"
|
||||
)
|
||||
assert reloaded_config.cloud_projects["work"].bisync_initialized is False
|
||||
|
||||
def test_backward_compatibility_loading_config_without_cloud_projects(self):
|
||||
@@ -352,11 +354,12 @@ class TestConfigManager:
|
||||
|
||||
# Manually write old-style config without cloud_projects
|
||||
import json
|
||||
|
||||
old_config_data = {
|
||||
"env": "dev",
|
||||
"projects": {"main": str(temp_path / "main")},
|
||||
"default_project": "main",
|
||||
"log_level": "INFO"
|
||||
"log_level": "INFO",
|
||||
}
|
||||
config_manager.config_file.write_text(json.dumps(old_config_data, indent=2))
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Test rclone configuration management."""
|
||||
|
||||
import configparser
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from basic_memory.cli.commands.cloud.rclone_config import (
|
||||
configure_rclone_remote,
|
||||
load_rclone_config,
|
||||
save_rclone_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_rclone_config(monkeypatch):
|
||||
"""Create a temporary rclone config directory."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
config_dir = Path(temp_dir) / ".config" / "rclone"
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
config_path = config_dir / "rclone.conf"
|
||||
|
||||
# Monkeypatch get_rclone_config_path to use temp directory
|
||||
monkeypatch.setattr(
|
||||
"basic_memory.cli.commands.cloud.rclone_config.get_rclone_config_path",
|
||||
lambda: config_path,
|
||||
)
|
||||
|
||||
yield config_path
|
||||
|
||||
|
||||
def test_configure_rclone_remote(temp_rclone_config):
|
||||
"""Test configuring simplified rclone remote."""
|
||||
# Configure remote
|
||||
remote_name = configure_rclone_remote(
|
||||
access_key="test_access_key",
|
||||
secret_key="test_secret_key",
|
||||
endpoint="https://test.endpoint.com",
|
||||
region="test-region",
|
||||
)
|
||||
|
||||
# Should return correct remote name
|
||||
assert remote_name == "basic-memory-cloud"
|
||||
|
||||
# Load and verify config
|
||||
config = load_rclone_config()
|
||||
|
||||
# Should have the remote section
|
||||
assert config.has_section("basic-memory-cloud")
|
||||
|
||||
# Should have correct settings
|
||||
assert config.get("basic-memory-cloud", "type") == "s3"
|
||||
assert config.get("basic-memory-cloud", "provider") == "Other"
|
||||
assert config.get("basic-memory-cloud", "access_key_id") == "test_access_key"
|
||||
assert config.get("basic-memory-cloud", "secret_access_key") == "test_secret_key"
|
||||
assert config.get("basic-memory-cloud", "endpoint") == "https://test.endpoint.com"
|
||||
assert config.get("basic-memory-cloud", "region") == "test-region"
|
||||
|
||||
|
||||
def test_configure_rclone_remote_default_values(temp_rclone_config):
|
||||
"""Test configuring remote with default endpoint and region."""
|
||||
remote_name = configure_rclone_remote(access_key="test_key", secret_key="test_secret")
|
||||
|
||||
assert remote_name == "basic-memory-cloud"
|
||||
|
||||
config = load_rclone_config()
|
||||
|
||||
# Should use default values
|
||||
assert config.get("basic-memory-cloud", "endpoint") == "https://fly.storage.tigris.dev"
|
||||
assert config.get("basic-memory-cloud", "region") == "auto"
|
||||
|
||||
|
||||
def test_configure_rclone_remote_updates_existing(temp_rclone_config):
|
||||
"""Test that configuring remote updates existing configuration."""
|
||||
# First configuration
|
||||
configure_rclone_remote(access_key="old_key", secret_key="old_secret")
|
||||
|
||||
# Update configuration
|
||||
configure_rclone_remote(access_key="new_key", secret_key="new_secret")
|
||||
|
||||
config = load_rclone_config()
|
||||
|
||||
# Should have updated values
|
||||
assert config.get("basic-memory-cloud", "access_key_id") == "new_key"
|
||||
assert config.get("basic-memory-cloud", "secret_access_key") == "new_secret"
|
||||
|
||||
# Should only have one section (not multiple)
|
||||
sections = config.sections()
|
||||
assert sections.count("basic-memory-cloud") == 1
|
||||
|
||||
|
||||
def test_save_and_load_rclone_config(temp_rclone_config):
|
||||
"""Test saving and loading rclone config."""
|
||||
# Create config
|
||||
config = configparser.ConfigParser()
|
||||
config.add_section("test-remote")
|
||||
config.set("test-remote", "type", "s3")
|
||||
config.set("test-remote", "provider", "AWS")
|
||||
|
||||
# Save config
|
||||
save_rclone_config(config)
|
||||
|
||||
# Load and verify
|
||||
loaded_config = load_rclone_config()
|
||||
assert loaded_config.has_section("test-remote")
|
||||
assert loaded_config.get("test-remote", "type") == "s3"
|
||||
assert loaded_config.get("test-remote", "provider") == "AWS"
|
||||
|
||||
|
||||
def test_load_rclone_config_nonexistent(temp_rclone_config):
|
||||
"""Test loading config when file doesn't exist."""
|
||||
# Delete the config file if it exists
|
||||
if temp_rclone_config.exists():
|
||||
temp_rclone_config.unlink()
|
||||
|
||||
# Should return empty config without error
|
||||
config = load_rclone_config()
|
||||
assert len(config.sections()) == 0
|
||||
Reference in New Issue
Block a user