diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81b82686..34587957 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,7 @@ jobs: test-sqlite-unit: name: Test SQLite Unit (${{ matrix.os }}, Python ${{ matrix.python-version }}) - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -147,7 +147,7 @@ jobs: test-postgres-unit: name: Test Postgres Unit (Python ${{ matrix.python-version }}) - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false matrix: diff --git a/docs/cloud-cli.md b/docs/cloud-cli.md index 8596d010..0314ed2c 100644 --- a/docs/cloud-cli.md +++ b/docs/cloud-cli.md @@ -582,33 +582,62 @@ bm cloud logout **Default patterns:** ```gitignore -# Version control -.git/** - -# Python -__pycache__/** -*.pyc -.venv/** -venv/** - -# Node.js -node_modules/** +# Hidden files and directories +.* # Basic Memory internals -memory.db/** -memory.db-shm/** -memory.db-wal/** -config.json/** -watch-status.json/** -.bmignore.rclone/** +*.db +*.db-shm +*.db-wal +config.json + +# Version control +.git +.svn + +# Python +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +*.egg-info +.tox +.mypy_cache +.ruff_cache + +# Virtual environments +.venv +venv +env +.env + +# Node.js +node_modules + +# Build artifacts +build +dist +.cache + +# IDE +.idea +.vscode # OS files -.DS_Store/** -Thumbs.db/** +.DS_Store +Thumbs.db +desktop.ini -# Environment files -.env/** -.env.local/** +# Obsidian +.obsidian + +# Temporary files +*.tmp +*.swp +*.swo +*~ ``` **How it works:** @@ -617,6 +646,11 @@ Thumbs.db/** 3. Rclone uses filters during sync 4. Same patterns used by all projects +During conversion, file patterns exclude the direct match and recursive contents. +For example, `config.json` becomes both `- config.json` and `- config.json/**`, +while `.*` becomes both `- .*` and `- .*/**`. Directory-only patterns keep +their trailing slash, so `cache/` becomes `- cache/` and `- cache/**`. + **Customizing:** ```bash @@ -624,7 +658,7 @@ Thumbs.db/** code ~/.basic-memory/.bmignore # Add custom patterns -echo "*.tmp/**" >> ~/.basic-memory/.bmignore +echo "*.tmp" >> ~/.basic-memory/.bmignore # Next sync uses updated patterns bm project bisync --name research diff --git a/src/basic_memory/cli/commands/cloud/bisync_commands.py b/src/basic_memory/cli/commands/cloud/bisync_commands.py index c7ad9139..4586848d 100644 --- a/src/basic_memory/cli/commands/cloud/bisync_commands.py +++ b/src/basic_memory/cli/commands/cloud/bisync_commands.py @@ -14,6 +14,24 @@ class BisyncError(Exception): pass +def _rclone_exclude_filters(pattern: str) -> list[str]: + """Return rclone exclude filters for a gitignore-style pattern.""" + if pattern.endswith("/"): + # Trigger: gitignore-style patterns ending in / are directory-only rules. + # Why: stripping the slash would also exclude a same-named file. + # Outcome: rclone keeps the directory rule and excludes recursive contents. + return [f"- {pattern}", f"- {pattern}**"] + + path_pattern = pattern.removesuffix("/**") + + # Trigger: rclone treats a directory contents filter separately from the + # directory/file path itself. + # Why: files like config.json and directory markers like .obsidian must both + # be excluded, along with anything below matching directories. + # Outcome: every ignore pattern excludes the direct match and recursive children. + return [f"- {path_pattern}", f"- {path_pattern}/**"] + + async def get_mount_info() -> TenantMountInfo: """Get current tenant information from cloud API.""" try: @@ -78,19 +96,11 @@ def convert_bmignore_to_rclone_filters() -> Path: patterns.append(line) continue - # Convert gitignore pattern to rclone filter syntax - # gitignore: node_modules → rclone: - node_modules/** - # gitignore: *.pyc → rclone: - *.pyc - if "*" in line: - # Pattern already has wildcard, just add exclude prefix - patterns.append(f"- {line}") - else: - # Directory pattern - add /** for recursive exclude - patterns.append(f"- {line}/**") + patterns.extend(_rclone_exclude_filters(line)) except Exception: # If we can't read the file, create a minimal filter - patterns = ["# Error reading .bmignore, using minimal filters", "- .git/**"] + patterns = ["# Error reading .bmignore, using minimal filters", "- .git", "- .git/**"] # Write rclone filter file rclone_filter_path.write_text("\n".join(patterns) + "\n") diff --git a/tests/cli/cloud/test_rclone_config_and_bmignore_filters.py b/tests/cli/cloud/test_rclone_config_and_bmignore_filters.py index 064f29d4..98bb5312 100644 --- a/tests/cli/cloud/test_rclone_config_and_bmignore_filters.py +++ b/tests/cli/cloud/test_rclone_config_and_bmignore_filters.py @@ -32,13 +32,48 @@ def test_convert_bmignore_to_rclone_filters_creates_and_converts(config_home): # Comments/empties preserved assert "# comment" in content assert "" in content - # Directory pattern becomes recursive exclude + # Plain and wildcard patterns exclude direct matches and recursive contents. + assert "- node_modules" in content assert "- node_modules/**" in content - # Wildcard pattern becomes simple exclude assert "- *.pyc" in content + assert "- *.pyc/**" in content + assert "- .git" in content assert "- .git/**" in content +def test_convert_bmignore_to_rclone_filters_excludes_files_and_hidden_directory_contents( + config_home, +): + bmignore = get_bmignore_path() + bmignore.parent.mkdir(parents=True, exist_ok=True) + bmignore.write_text("config.json\n.*\nnode_modules/**\n", encoding="utf-8") + + rclone_filter = convert_bmignore_to_rclone_filters() + content = rclone_filter.read_text(encoding="utf-8").splitlines() + + assert "- config.json" in content + assert "- config.json/**" in content + assert "- .*" in content + assert "- .*/**" in content + assert "- node_modules" in content + assert "- node_modules/**" in content + + +def test_convert_bmignore_to_rclone_filters_preserves_directory_only_patterns(config_home): + bmignore = get_bmignore_path() + bmignore.parent.mkdir(parents=True, exist_ok=True) + bmignore.write_text("cache/\nconfig.json/**\n", encoding="utf-8") + + rclone_filter = convert_bmignore_to_rclone_filters() + content = rclone_filter.read_text(encoding="utf-8").splitlines() + + assert "- cache/" in content + assert "- cache/**" in content + assert "- cache" not in content + assert "- config.json" in content + assert "- config.json/**" in content + + def test_convert_bmignore_to_rclone_filters_is_cached_when_up_to_date(config_home): bmignore = get_bmignore_path() bmignore.parent.mkdir(parents=True, exist_ok=True)