mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Drew Cain
0ba6f219f1
Signed-off-by: phernandez <paul@basicmachines.co> Signed-off-by: Claude <noreply@anthropic.com> Signed-off-by: Paul Hernandez <60959+phernandez@users.noreply.github.com> Co-authored-by: phernandez <paul@basicmachines.co> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Paul Hernandez <60959+phernandez@users.noreply.github.com>
791 lines
26 KiB
Python
791 lines
26 KiB
Python
"""Tests for incremental scan watermark optimization (Phase 1.5).
|
|
|
|
These tests verify the scan watermark feature that dramatically improves sync
|
|
performance on large projects by:
|
|
- Using find -newermt for incremental scans (only changed files)
|
|
- Tracking last_scan_timestamp and last_file_count
|
|
- Falling back to full scan when deletions detected
|
|
|
|
Expected performance improvements:
|
|
- No changes: 225x faster (2s vs 450s for 1,460 files)
|
|
- Few changes: 84x faster (5s vs 420s)
|
|
"""
|
|
|
|
import time
|
|
from pathlib import Path
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
|
|
from basic_memory.config import ProjectConfig
|
|
from basic_memory.sync.sync_service import SyncService
|
|
|
|
|
|
async def create_test_file(path: Path, content: str = "test content") -> None:
|
|
"""Create a test file with given content."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(content)
|
|
|
|
|
|
async def sleep_past_watermark(duration: float = 1.1) -> None:
|
|
"""Sleep long enough to ensure mtime is newer than watermark.
|
|
|
|
Args:
|
|
duration: Sleep duration in seconds (default 1.1s for filesystem precision)
|
|
"""
|
|
time.sleep(duration)
|
|
|
|
|
|
# ==============================================================================
|
|
# Scan Strategy Selection Tests
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_first_sync_uses_full_scan(sync_service: SyncService, project_config: ProjectConfig):
|
|
"""Test that first sync (no watermark) triggers full scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create test files
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nContent 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2\nContent 2")
|
|
|
|
# First sync - should use full scan (no watermark exists)
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.new) == 2
|
|
assert "file1.md" in report.new
|
|
assert "file2.md" in report.new
|
|
|
|
# Verify watermark was set
|
|
project = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project.last_scan_timestamp is not None
|
|
assert project.last_file_count >= 2 # May include config files
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_count_decreased_triggers_full_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that file deletion (count decreased) triggers full scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
await create_test_file(project_dir / "file3.md", "# File 3")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Delete a file
|
|
(project_dir / "file2.md").unlink()
|
|
|
|
# Sleep to ensure file operations complete
|
|
await sleep_past_watermark()
|
|
|
|
# Second sync - should detect deletion via full scan (file count decreased)
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.deleted) == 1
|
|
assert "file2.md" in report.deleted
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_count_same_uses_incremental_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that same file count uses incremental scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nOriginal")
|
|
await create_test_file(project_dir / "file2.md", "# File 2\nOriginal")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer than watermark
|
|
await sleep_past_watermark()
|
|
|
|
# Modify one file (file count stays the same)
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nModified")
|
|
|
|
# Second sync - should use incremental scan (same file count)
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.modified) == 1
|
|
assert "file1.md" in report.modified
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_file_count_increased_uses_incremental_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that increased file count still uses incremental scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer than watermark
|
|
await sleep_past_watermark()
|
|
|
|
# Add a new file (file count increased)
|
|
await create_test_file(project_dir / "file3.md", "# File 3")
|
|
|
|
# Second sync - should use incremental scan and detect new file
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.new) == 1
|
|
assert "file3.md" in report.new
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_force_full_bypasses_watermark_optimization(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that force_full=True bypasses watermark optimization and scans all files.
|
|
|
|
This is critical for detecting changes made by external tools like rclone bisync
|
|
that don't update mtimes detectably. See issue #407.
|
|
"""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nOriginal")
|
|
await create_test_file(project_dir / "file2.md", "# File 2\nOriginal")
|
|
|
|
# First sync - establishes watermark
|
|
report = await sync_service.sync(project_dir)
|
|
assert len(report.new) == 2
|
|
|
|
# Verify watermark was set
|
|
project = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project.last_scan_timestamp is not None
|
|
initial_timestamp = project.last_scan_timestamp
|
|
|
|
# Sleep to ensure time passes
|
|
await sleep_past_watermark()
|
|
|
|
# Modify a file WITHOUT updating mtime (simulates external tool like rclone)
|
|
# We set mtime to be BEFORE the watermark to ensure incremental scan won't detect it
|
|
file_path = project_dir / "file1.md"
|
|
file_path.stat()
|
|
await create_test_file(file_path, "# File 1\nModified by external tool")
|
|
|
|
# Set mtime to be before the watermark (use time from before first sync)
|
|
# This simulates rclone bisync which may preserve original timestamps
|
|
import os
|
|
|
|
old_time = initial_timestamp - 10 # 10 seconds before watermark
|
|
os.utime(file_path, (old_time, old_time))
|
|
|
|
# Normal incremental sync should NOT detect the change (mtime before watermark)
|
|
report = await sync_service.sync(project_dir)
|
|
assert len(report.modified) == 0, (
|
|
"Incremental scan should not detect changes with mtime older than watermark"
|
|
)
|
|
|
|
# Force full scan should detect the change via checksum comparison
|
|
report = await sync_service.sync(project_dir, force_full=True)
|
|
assert len(report.modified) == 1, "Force full scan should detect changes via checksum"
|
|
assert "file1.md" in report.modified
|
|
|
|
# Verify watermark was still updated after force_full
|
|
project = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project.last_scan_timestamp is not None
|
|
assert project.last_scan_timestamp > initial_timestamp
|
|
|
|
|
|
# ==============================================================================
|
|
# Incremental Scan Base Cases
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_incremental_scan_no_changes(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan with no changes returns empty report."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure time passes
|
|
await sleep_past_watermark()
|
|
|
|
# Second sync - no changes
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.new) == 0
|
|
assert len(report.modified) == 0
|
|
assert len(report.deleted) == 0
|
|
assert len(report.moves) == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_incremental_scan_detects_new_file(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan detects newly created files."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial file
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer than watermark
|
|
await sleep_past_watermark()
|
|
|
|
# Create new file
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
|
|
# Second sync - should detect new file via incremental scan
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.new) == 1
|
|
assert "file2.md" in report.new
|
|
assert len(report.modified) == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_incremental_scan_detects_modified_file(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan detects modified files."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
file_path = project_dir / "file1.md"
|
|
await create_test_file(file_path, "# File 1\nOriginal content")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer than watermark
|
|
await sleep_past_watermark()
|
|
|
|
# Modify the file
|
|
await create_test_file(file_path, "# File 1\nModified content")
|
|
|
|
# Second sync - should detect modification via incremental scan
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.modified) == 1
|
|
assert "file1.md" in report.modified
|
|
assert len(report.new) == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_incremental_scan_detects_multiple_changes(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan detects multiple file changes."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nOriginal")
|
|
await create_test_file(project_dir / "file2.md", "# File 2\nOriginal")
|
|
await create_test_file(project_dir / "file3.md", "# File 3\nOriginal")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer than watermark
|
|
await sleep_past_watermark()
|
|
|
|
# Modify multiple files
|
|
await create_test_file(project_dir / "file1.md", "# File 1\nModified")
|
|
await create_test_file(project_dir / "file3.md", "# File 3\nModified")
|
|
await create_test_file(project_dir / "file4.md", "# File 4\nNew")
|
|
|
|
# Second sync - should detect all changes via incremental scan
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.modified) == 2
|
|
assert "file1.md" in report.modified
|
|
assert "file3.md" in report.modified
|
|
assert len(report.new) == 1
|
|
assert "file4.md" in report.new
|
|
|
|
|
|
# ==============================================================================
|
|
# Deletion Detection Tests
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deletion_triggers_full_scan_single_file(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that deleting a single file triggers full scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
await create_test_file(project_dir / "file3.md", "# File 3")
|
|
|
|
# First sync
|
|
report1 = await sync_service.sync(project_dir)
|
|
assert len(report1.new) == 3
|
|
|
|
# Delete one file
|
|
(project_dir / "file2.md").unlink()
|
|
|
|
# Sleep to ensure file operations complete
|
|
await sleep_past_watermark()
|
|
|
|
# Second sync - should trigger full scan due to decreased file count
|
|
report2 = await sync_service.sync(project_dir)
|
|
|
|
assert len(report2.deleted) == 1
|
|
assert "file2.md" in report2.deleted
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deletion_triggers_full_scan_multiple_files(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that deleting multiple files triggers full scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
await create_test_file(project_dir / "file3.md", "# File 3")
|
|
await create_test_file(project_dir / "file4.md", "# File 4")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Delete multiple files
|
|
(project_dir / "file2.md").unlink()
|
|
(project_dir / "file4.md").unlink()
|
|
|
|
# Sleep to ensure file operations complete
|
|
await sleep_past_watermark()
|
|
|
|
# Second sync - should trigger full scan and detect both deletions
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.deleted) == 2
|
|
assert "file2.md" in report.deleted
|
|
assert "file4.md" in report.deleted
|
|
|
|
|
|
# ==============================================================================
|
|
# Move Detection Tests
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_move_detection_requires_full_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that file moves require full scan to be detected (cannot detect in incremental).
|
|
|
|
Moves (renames) don't update mtime, so incremental scans can't detect them.
|
|
To trigger a full scan for move detection, we need file count to decrease.
|
|
This test verifies moves are detected when combined with a deletion.
|
|
"""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files - include extra file to delete later
|
|
old_path = project_dir / "old" / "file.md"
|
|
content = dedent(
|
|
"""
|
|
---
|
|
title: Test File
|
|
type: note
|
|
---
|
|
# Test File
|
|
Distinctive content for move detection
|
|
"""
|
|
).strip()
|
|
await create_test_file(old_path, content)
|
|
await create_test_file(project_dir / "other.md", "# Other\nContent")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure operations complete and watermark is in the past
|
|
await sleep_past_watermark()
|
|
|
|
# Move file AND delete another to trigger full scan
|
|
# Move alone won't work because file count stays same (no full scan)
|
|
new_path = project_dir / "new" / "moved.md"
|
|
new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
old_path.rename(new_path)
|
|
(project_dir / "other.md").unlink() # Delete to trigger full scan
|
|
|
|
# Second sync - full scan due to deletion, move detected via checksum
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.moves) == 1
|
|
assert "old/file.md" in report.moves
|
|
assert report.moves["old/file.md"] == "new/moved.md"
|
|
assert len(report.deleted) == 1
|
|
assert "other.md" in report.deleted
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_move_detection_in_full_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that file moves are detected via checksum in full scan."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
old_path = project_dir / "old" / "file.md"
|
|
content = dedent(
|
|
"""
|
|
---
|
|
title: Test File
|
|
type: note
|
|
---
|
|
# Test File
|
|
Distinctive content for move detection
|
|
"""
|
|
).strip()
|
|
await create_test_file(old_path, content)
|
|
await create_test_file(project_dir / "other.md", "# Other\nContent")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure operations complete
|
|
await sleep_past_watermark()
|
|
|
|
# Move file AND delete another to trigger full scan
|
|
new_path = project_dir / "new" / "moved.md"
|
|
new_path.parent.mkdir(parents=True, exist_ok=True)
|
|
old_path.rename(new_path)
|
|
(project_dir / "other.md").unlink()
|
|
|
|
# Second sync - full scan due to deletion, should still detect move
|
|
report = await sync_service.sync(project_dir)
|
|
|
|
assert len(report.moves) == 1
|
|
assert "old/file.md" in report.moves
|
|
assert report.moves["old/file.md"] == "new/moved.md"
|
|
assert len(report.deleted) == 1
|
|
assert "other.md" in report.deleted
|
|
|
|
|
|
# ==============================================================================
|
|
# Watermark Update Tests
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watermark_updated_after_successful_sync(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that watermark is updated after each successful sync."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial file
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
|
|
# Get project before sync
|
|
project_before = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project_before.last_scan_timestamp is None
|
|
assert project_before.last_file_count is None
|
|
|
|
# First sync
|
|
sync_start = time.time()
|
|
await sync_service.sync(project_dir)
|
|
sync_end = time.time()
|
|
|
|
# Verify watermark was set
|
|
project_after = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project_after.last_scan_timestamp is not None
|
|
assert project_after.last_file_count >= 1 # May include config files
|
|
|
|
# Watermark should be between sync start and end
|
|
assert sync_start <= project_after.last_scan_timestamp <= sync_end
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watermark_uses_sync_start_time(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that watermark uses sync start time, not end time."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial file
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
|
|
# First sync - capture timestamps
|
|
sync_start = time.time()
|
|
await sync_service.sync(project_dir)
|
|
sync_end = time.time()
|
|
|
|
# Get watermark
|
|
project = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
|
|
# Watermark should be closer to start than end
|
|
# (In practice, watermark == sync_start_timestamp captured in sync())
|
|
time_from_start = abs(project.last_scan_timestamp - sync_start)
|
|
time_from_end = abs(project.last_scan_timestamp - sync_end)
|
|
|
|
assert time_from_start < time_from_end
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watermark_file_count_accurate(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that watermark file count accurately reflects synced files."""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial files
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
await create_test_file(project_dir / "file2.md", "# File 2")
|
|
await create_test_file(project_dir / "file3.md", "# File 3")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Verify file count
|
|
project1 = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
initial_count = project1.last_file_count
|
|
assert initial_count >= 3 # May include config files
|
|
|
|
# Add more files
|
|
await sleep_past_watermark()
|
|
await create_test_file(project_dir / "file4.md", "# File 4")
|
|
await create_test_file(project_dir / "file5.md", "# File 5")
|
|
|
|
# Second sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Verify updated count increased by 2
|
|
project2 = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project2.last_file_count == initial_count + 2
|
|
|
|
|
|
# ==============================================================================
|
|
# Edge Cases and Error Handling
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_concurrent_file_changes_handled_gracefully(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that files created/modified during sync are handled correctly.
|
|
|
|
Files created during sync (between start and file processing) should be
|
|
caught in the next sync, not cause errors in the current sync.
|
|
"""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial file
|
|
await create_test_file(project_dir / "file1.md", "# File 1")
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Sleep to ensure mtime will be newer
|
|
await sleep_past_watermark()
|
|
|
|
# Create file that will have mtime very close to watermark
|
|
# In real scenarios, this could be created during sync
|
|
await create_test_file(project_dir / "concurrent.md", "# Concurrent")
|
|
|
|
# Should be caught in next sync without errors
|
|
report = await sync_service.sync(project_dir)
|
|
assert "concurrent.md" in report.new
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_directory_handles_incremental_scan(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan handles empty directories correctly."""
|
|
project_dir = project_config.home
|
|
|
|
# First sync with empty directory (no user files)
|
|
report1 = await sync_service.sync(project_dir)
|
|
assert len(report1.new) == 0
|
|
|
|
# Verify watermark was set even for empty directory
|
|
project = await sync_service.project_repository.find_by_id(
|
|
sync_service.entity_repository.project_id
|
|
)
|
|
assert project.last_scan_timestamp is not None
|
|
# May have config files, so just check it's set
|
|
assert project.last_file_count is not None
|
|
|
|
# Second sync - still empty (no new user files)
|
|
report2 = await sync_service.sync(project_dir)
|
|
assert len(report2.new) == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_incremental_scan_respects_gitignore(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that incremental scan respects .gitignore patterns."""
|
|
project_dir = project_config.home
|
|
|
|
# Create .gitignore
|
|
(project_dir / ".gitignore").write_text("*.ignored\n.hidden/\n")
|
|
|
|
# Reload ignore patterns
|
|
from basic_memory.ignore_utils import load_gitignore_patterns
|
|
|
|
sync_service._ignore_patterns = load_gitignore_patterns(project_dir)
|
|
|
|
# Create files - some should be ignored
|
|
await create_test_file(project_dir / "included.md", "# Included")
|
|
await create_test_file(project_dir / "excluded.ignored", "# Excluded")
|
|
|
|
# First sync
|
|
report1 = await sync_service.sync(project_dir)
|
|
assert "included.md" in report1.new
|
|
assert "excluded.ignored" not in report1.new
|
|
|
|
# Sleep and add more files
|
|
await sleep_past_watermark()
|
|
await create_test_file(project_dir / "included2.md", "# Included 2")
|
|
await create_test_file(project_dir / "excluded2.ignored", "# Excluded 2")
|
|
|
|
# Second sync - incremental scan should also respect ignore patterns
|
|
report2 = await sync_service.sync(project_dir)
|
|
assert "included2.md" in report2.new
|
|
assert "excluded2.ignored" not in report2.new
|
|
|
|
|
|
# ==============================================================================
|
|
# Relation Resolution Optimization Tests
|
|
# ==============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_relation_resolution_skipped_when_no_changes(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that relation resolution is skipped when no file changes detected.
|
|
|
|
This optimization prevents wasting time resolving relations when there are
|
|
no changes, dramatically improving sync performance for large projects.
|
|
"""
|
|
project_dir = project_config.home
|
|
|
|
# Create initial file with wikilink
|
|
content = dedent(
|
|
"""
|
|
---
|
|
title: File with Link
|
|
type: note
|
|
---
|
|
# File with Link
|
|
This links to [[Target File]]
|
|
"""
|
|
).strip()
|
|
await create_test_file(project_dir / "file1.md", content)
|
|
|
|
# First sync - will resolve relations (or leave unresolved)
|
|
report1 = await sync_service.sync(project_dir)
|
|
assert len(report1.new) == 1
|
|
|
|
# Check that there are unresolved relations (target doesn't exist)
|
|
unresolved = await sync_service.relation_repository.find_unresolved_relations()
|
|
unresolved_count_before = len(unresolved)
|
|
assert unresolved_count_before > 0 # Should have unresolved relation to [[Target File]]
|
|
|
|
# Sleep to ensure time passes
|
|
await sleep_past_watermark()
|
|
|
|
# Second sync - no changes, should skip relation resolution
|
|
report2 = await sync_service.sync(project_dir)
|
|
assert report2.total == 0 # No changes detected
|
|
|
|
# Verify unresolved relations count unchanged (resolution was skipped)
|
|
unresolved_after = await sync_service.relation_repository.find_unresolved_relations()
|
|
assert len(unresolved_after) == unresolved_count_before
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_relation_resolution_runs_when_files_modified(
|
|
sync_service: SyncService, project_config: ProjectConfig
|
|
):
|
|
"""Test that relation resolution runs when files are actually modified."""
|
|
project_dir = project_config.home
|
|
|
|
# Create file with unresolved wikilink
|
|
content1 = dedent(
|
|
"""
|
|
---
|
|
title: File with Link
|
|
type: note
|
|
---
|
|
# File with Link
|
|
This links to [[Target File]]
|
|
"""
|
|
).strip()
|
|
await create_test_file(project_dir / "file1.md", content1)
|
|
|
|
# First sync
|
|
await sync_service.sync(project_dir)
|
|
|
|
# Verify unresolved relation exists
|
|
unresolved_before = await sync_service.relation_repository.find_unresolved_relations()
|
|
assert len(unresolved_before) > 0
|
|
|
|
# Sleep to ensure mtime will be newer
|
|
await sleep_past_watermark()
|
|
|
|
# Create the target file (should resolve the relation)
|
|
content2 = dedent(
|
|
"""
|
|
---
|
|
title: Target File
|
|
type: note
|
|
---
|
|
# Target File
|
|
This is the target.
|
|
"""
|
|
).strip()
|
|
await create_test_file(project_dir / "target.md", content2)
|
|
|
|
# Second sync - should detect new file and resolve relations
|
|
report = await sync_service.sync(project_dir)
|
|
assert len(report.new) == 1
|
|
assert "target.md" in report.new
|
|
|
|
# Verify relation was resolved (unresolved count decreased)
|
|
unresolved_after = await sync_service.relation_repository.find_unresolved_relations()
|
|
assert len(unresolved_after) < len(unresolved_before)
|