mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix: skip archive files during cloud upload (#420)
Signed-off-by: phernandez <paul@basicmachines.co> Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Paul Hernandez <phernandez@users.noreply.github.com>
This commit is contained in:
@@ -10,6 +10,9 @@ from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_pat
|
||||
from basic_memory.mcp.async_client import get_client
|
||||
from basic_memory.mcp.tools.utils import call_put
|
||||
|
||||
# Archive file extensions that should be skipped during upload
|
||||
ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".tgz", ".tbz2"}
|
||||
|
||||
|
||||
async def upload_path(
|
||||
local_path: Path,
|
||||
@@ -61,11 +64,18 @@ async def upload_path(
|
||||
|
||||
# Calculate total size
|
||||
total_bytes = sum(file_path.stat().st_size for file_path, _ in files_to_upload)
|
||||
skipped_count = 0
|
||||
|
||||
# If dry run, just show what would be uploaded
|
||||
if dry_run:
|
||||
print("\nFiles that would be uploaded:")
|
||||
for file_path, relative_path in files_to_upload:
|
||||
# Skip archive files
|
||||
if _is_archive_file(file_path):
|
||||
print(f" [SKIP] {relative_path} (archive file)")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
size = file_path.stat().st_size
|
||||
if size < 1024:
|
||||
size_str = f"{size} bytes"
|
||||
@@ -78,6 +88,12 @@ async def upload_path(
|
||||
# Upload files using httpx
|
||||
async with get_client() as client:
|
||||
for i, (file_path, relative_path) in enumerate(files_to_upload, 1):
|
||||
# Skip archive files (zip, tar, gz, etc.)
|
||||
if _is_archive_file(file_path):
|
||||
print(f"Skipping archive file: {relative_path} ({i}/{len(files_to_upload)})")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# Build remote path: /webdav/{project_name}/{relative_path}
|
||||
remote_path = f"/webdav/{project_name}/{relative_path}"
|
||||
print(f"Uploading {relative_path} ({i}/{len(files_to_upload)})")
|
||||
@@ -105,10 +121,15 @@ async def upload_path(
|
||||
else:
|
||||
size_str = f"{total_bytes / (1024 * 1024):.1f} MB"
|
||||
|
||||
uploaded_count = len(files_to_upload) - skipped_count
|
||||
if dry_run:
|
||||
print(f"\nTotal: {len(files_to_upload)} file(s) ({size_str})")
|
||||
print(f"\nTotal: {uploaded_count} file(s) ({size_str})")
|
||||
if skipped_count > 0:
|
||||
print(f" Would skip {skipped_count} archive file(s)")
|
||||
else:
|
||||
print(f"Upload complete: {len(files_to_upload)} file(s) ({size_str})")
|
||||
print(f"✓ Upload complete: {uploaded_count} file(s) ({size_str})")
|
||||
if skipped_count > 0:
|
||||
print(f" Skipped {skipped_count} archive file(s)")
|
||||
|
||||
return True
|
||||
|
||||
@@ -120,6 +141,19 @@ async def upload_path(
|
||||
return False
|
||||
|
||||
|
||||
def _is_archive_file(file_path: Path) -> bool:
|
||||
"""
|
||||
Check if a file is an archive file based on its extension.
|
||||
|
||||
Args:
|
||||
file_path: Path to the file to check
|
||||
|
||||
Returns:
|
||||
True if file is an archive, False otherwise
|
||||
"""
|
||||
return file_path.suffix.lower() in ARCHIVE_EXTENSIONS
|
||||
|
||||
|
||||
def _get_files_to_upload(
|
||||
directory: Path, verbose: bool = False, use_gitignore: bool = True
|
||||
) -> list[tuple[Path, str]]:
|
||||
|
||||
@@ -327,6 +327,55 @@ class TestUploadPath:
|
||||
call_args = mock_put.call_args
|
||||
assert call_args[0][1] == "/webdav/my-project/subdir/file.txt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_archive_files(self, tmp_path, capsys):
|
||||
"""Test that archive files are skipped during upload."""
|
||||
# Create test files including archives
|
||||
(tmp_path / "notes.md").write_text("content")
|
||||
(tmp_path / "backup.zip").write_text("fake zip")
|
||||
(tmp_path / "data.tar.gz").write_text("fake tar")
|
||||
|
||||
mock_client = AsyncMock()
|
||||
mock_response = Mock()
|
||||
mock_response.raise_for_status = Mock()
|
||||
|
||||
with patch("basic_memory.cli.commands.cloud.upload.get_client") as mock_get_client:
|
||||
with patch("basic_memory.cli.commands.cloud.upload.call_put") as mock_put:
|
||||
with patch(
|
||||
"basic_memory.cli.commands.cloud.upload._get_files_to_upload"
|
||||
) as mock_get_files:
|
||||
with patch("aiofiles.open", create=True) as mock_aiofiles_open:
|
||||
mock_get_client.return_value.__aenter__.return_value = mock_client
|
||||
mock_get_client.return_value.__aexit__.return_value = None
|
||||
mock_put.return_value = mock_response
|
||||
|
||||
# Mock file listing with all files
|
||||
mock_get_files.return_value = [
|
||||
(tmp_path / "notes.md", "notes.md"),
|
||||
(tmp_path / "backup.zip", "backup.zip"),
|
||||
(tmp_path / "data.tar.gz", "data.tar.gz"),
|
||||
]
|
||||
|
||||
mock_file = AsyncMock()
|
||||
mock_file.read.return_value = b"content"
|
||||
mock_aiofiles_open.return_value.__aenter__.return_value = mock_file
|
||||
|
||||
result = await upload_path(tmp_path, "test-project")
|
||||
|
||||
# Should succeed
|
||||
assert result is True
|
||||
|
||||
# Should only upload the .md file (not the archives)
|
||||
assert mock_put.call_count == 1
|
||||
call_args = mock_put.call_args
|
||||
assert "notes.md" in call_args[0][1]
|
||||
|
||||
# Check output mentions skipping
|
||||
captured = capsys.readouterr()
|
||||
assert "Skipping archive file" in captured.out
|
||||
assert "backup.zip" in captured.out
|
||||
assert "Skipped 2 archive file(s)" in captured.out
|
||||
|
||||
def test_no_gitignore_skips_gitignore_patterns(self, tmp_path):
|
||||
"""Test that --no-gitignore flag skips .gitignore patterns."""
|
||||
# Create test files
|
||||
|
||||
Reference in New Issue
Block a user