From e78345ff250410b52506a39ea75cc99e5cfc1fd3 Mon Sep 17 00:00:00 2001 From: Paul Hernandez <60959+phernandez@users.noreply.github.com> Date: Tue, 21 Oct 2025 09:03:59 -0500 Subject: [PATCH] feat: Streaming Foundation & Async I/O Consolidation (SPEC-19) (#384) Signed-off-by: phernandez Co-authored-by: Claude --- pyproject.toml | 1 + ...ync Performance and Memory Optimization.md | 1046 +++++++++++++++++ ...5_add_mtime_and_size_columns_to_entity_.py | 49 + ..._add_scan_watermark_tracking_to_project.py | 37 + src/basic_memory/deps.py | 2 + src/basic_memory/file_utils.py | 91 +- src/basic_memory/mcp/tools/__init__.py | 2 - src/basic_memory/mcp/tools/build_context.py | 22 - src/basic_memory/mcp/tools/read_note.py | 8 - src/basic_memory/mcp/tools/sync_status.py | 261 ---- src/basic_memory/mcp/tools/utils.py | 70 -- src/basic_memory/mcp/tools/write_note.py | 9 - src/basic_memory/models/knowledge.py | 7 + src/basic_memory/models/project.py | 5 + .../repository/entity_repository.py | 32 + src/basic_memory/services/file_service.py | 186 ++- src/basic_memory/services/initialization.py | 10 - .../services/sync_status_service.py | 198 ---- src/basic_memory/sync/sync_service.py | 708 +++++++---- src/basic_memory/sync/watch_service.py | 9 +- src/basic_memory/utils.py | 15 + .../test_disable_permalinks_integration.py | 3 + tests/conftest.py | 2 + ...test_permalink_collision_file_overwrite.py | 24 + tests/mcp/test_tool_sync_status.py | 170 --- tests/mcp/test_tool_utils.py | 83 +- tests/repository/test_entity_repository.py | 166 +++ tests/services/test_project_service.py | 2 +- tests/services/test_sync_status_service.py | 262 ----- tests/sync/test_sync_service.py | 363 +++++- tests/sync/test_sync_service_incremental.py | 729 ++++++++++++ tests/sync/test_sync_wikilink_issue.py | 20 + tests/utils/test_file_utils.py | 113 -- tests/utils/test_utf8_handling.py | 12 +- uv.lock | 214 ++++ 35 files changed, 3336 insertions(+), 1595 deletions(-) create mode 100644 specs/SPEC-19 Sync Performance and Memory Optimization.md create mode 100644 src/basic_memory/alembic/versions/9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py create mode 100644 src/basic_memory/alembic/versions/e7e1f4367280_add_scan_watermark_tracking_to_project.py delete mode 100644 src/basic_memory/mcp/tools/sync_status.py delete mode 100644 src/basic_memory/services/sync_status_service.py delete mode 100644 tests/mcp/test_tool_sync_status.py delete mode 100644 tests/services/test_sync_status_service.py create mode 100644 tests/sync/test_sync_service_incremental.py diff --git a/pyproject.toml b/pyproject.toml index 100b298e..17defc51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "python-dotenv>=1.1.0", "pytest-aio>=1.9.0", "aiofiles>=24.1.0", # Async file I/O + "logfire>=0.73.0", # Optional observability (disabled by default via config) ] diff --git a/specs/SPEC-19 Sync Performance and Memory Optimization.md b/specs/SPEC-19 Sync Performance and Memory Optimization.md new file mode 100644 index 00000000..cf8dd735 --- /dev/null +++ b/specs/SPEC-19 Sync Performance and Memory Optimization.md @@ -0,0 +1,1046 @@ +--- +title: 'SPEC-19: Sync Performance and Memory Optimization' +type: spec +permalink: specs/spec-17-sync-performance-optimization +tags: +- performance +- memory +- sync +- optimization +- core +status: draft +--- + +# SPEC-19: Sync Performance and Memory Optimization + +## Why + +### Problem Statement + +Current sync implementation causes Out-of-Memory (OOM) kills and poor performance on production systems: + +**Evidence from Production**: +- **Tenant-6d2ff1a3**: OOM killed on 1GB machine + - Files: 2,621 total (31 PDFs, 80MB binary data) + - Memory: 1.5-1.7GB peak usage + - Sync duration: 15+ minutes + - Error: `Out of memory: Killed process 693 (python)` + +**Root Causes**: + +1. **Checksum-based scanning loads ALL files into memory** + - `scan_directory()` computes checksums for ALL 2,624 files upfront + - Results stored in multiple dicts (`ScanResult.files`, `SyncReport.checksums`) + - Even unchanged files are fully read and checksummed + +2. **Large files read entirely for checksums** + - 16MB PDF → Full read into memory → Compute checksum + - No streaming or chunked processing + - TigrisFS caching compounds memory usage + +3. **Unbounded concurrency** + - All 2,624 files processed simultaneously + - Each file loads full content into memory + - No semaphore limiting concurrent operations + +4. **Cloud-specific resource leaks** + - aiohttp session leak in keepalive (not in context manager) + - Circuit breaker resets every 30s sync cycle (ineffective) + - Thundering herd: all tenants sync at :00 and :30 + +### Impact + +- **Production stability**: OOM kills are unacceptable +- **User experience**: 15+ minute syncs are too slow +- **Cost**: Forced upgrades from 1GB → 2GB machines ($5-10/mo per tenant) +- **Scalability**: Current approach won't scale to 100+ tenants + +### Architectural Decision + +**Fix in basic-memory core first, NOT UberSync** + +Rationale: +- Root causes are algorithmic, not architectural +- Benefits all users (CLI + Cloud) +- Lower risk than new centralized service +- Known solutions (rsync/rclone use same pattern) +- Can defer UberSync until metrics prove it necessary + +## What + +### Affected Components + +**basic-memory (core)**: +- `src/basic_memory/sync/sync_service.py` - Core sync algorithm (~42KB) +- `src/basic_memory/models.py` - Entity model (add mtime/size columns) +- `src/basic_memory/file_utils.py` - Checksum computation functions +- `src/basic_memory/repository/entity_repository.py` - Database queries +- `alembic/versions/` - Database migration for schema changes + +**basic-memory-cloud (wrapper)**: +- `apps/api/src/basic_memory_cloud_api/sync_worker.py` - Cloud sync wrapper +- Circuit breaker implementation +- Sync coordination logic + +### Database Schema Changes + +Add to Entity model: +```python +mtime: float # File modification timestamp +size: int # File size in bytes +``` + +## How (High Level) + +### Phase 1: Core Algorithm Fixes (basic-memory) + +**Priority: P0 - Critical** + +#### 1.1 mtime-based Scanning (Issue #383) + +Replace expensive checksum-based scanning with lightweight stat-based comparison: + +```python +async def scan_directory(self, directory: Path) -> ScanResult: + """Scan using mtime/size instead of checksums""" + result = ScanResult() + + for root, dirnames, filenames in os.walk(str(directory)): + for filename in filenames: + rel_path = path.relative_to(directory).as_posix() + stat = path.stat() + + # Store lightweight metadata instead of checksum + result.files[rel_path] = { + 'mtime': stat.st_mtime, + 'size': stat.st_size + } + + return result + +async def scan(self, directory: Path): + """Compare mtime/size, only compute checksums for changed files""" + db_state = await self.get_db_file_state() # Include mtime/size + scan_result = await self.scan_directory(directory) + + for file_path, metadata in scan_result.files.items(): + db_metadata = db_state.get(file_path) + + # Only compute expensive checksum if mtime/size changed + if not db_metadata or metadata['mtime'] != db_metadata['mtime']: + checksum = await self._compute_checksum_streaming(file_path) + # Process immediately, don't accumulate in memory +``` + +**Benefits**: +- No file reads during initial scan (just stat calls) +- ~90% reduction in memory usage +- ~10x faster scan phase +- Only checksum files that actually changed + +#### 1.2 Streaming Checksum Computation (Issue #382) + +For large files (>1MB), use chunked reading to avoid loading entire file: + +```python +async def _compute_checksum_streaming(self, path: Path, chunk_size: int = 65536) -> str: + """Compute checksum using 64KB chunks for large files""" + hasher = hashlib.sha256() + + loop = asyncio.get_event_loop() + + def read_chunks(): + with open(path, 'rb') as f: + while chunk := f.read(chunk_size): + hasher.update(chunk) + + await loop.run_in_executor(None, read_chunks) + return hasher.hexdigest() + +async def _compute_checksum_async(self, file_path: Path) -> str: + """Choose appropriate checksum method based on file size""" + stat = file_path.stat() + + if stat.st_size > 1_048_576: # 1MB threshold + return await self._compute_checksum_streaming(file_path) + else: + # Small files: existing fast path + content = await self._read_file_async(file_path) + return compute_checksum(content) +``` + +**Benefits**: +- Constant memory usage regardless of file size +- 16MB PDF uses 64KB memory (not 16MB) +- Works well with TigrisFS network I/O + +#### 1.3 Bounded Concurrency (Issue #198) + +Add semaphore to limit concurrent file operations, or consider using aiofiles and async reads + +```python +class SyncService: + def __init__(self, ...): + # ... existing code ... + self._file_semaphore = asyncio.Semaphore(10) # Max 10 concurrent + self._max_tracked_failures = 100 # LRU cache limit + + async def _read_file_async(self, file_path: Path) -> str: + async with self._file_semaphore: + loop = asyncio.get_event_loop() + return await loop.run_in_executor( + self._thread_pool, + file_path.read_text, + "utf-8" + ) + + async def _record_failure(self, path: str, error: str): + # ... existing code ... + + # Implement LRU eviction + if len(self._file_failures) > self._max_tracked_failures: + self._file_failures.popitem(last=False) # Remove oldest +``` + +**Benefits**: +- Maximum 10 files in memory at once (vs all 2,624) +- 90%+ reduction in peak memory usage +- Prevents unbounded memory growth on error-prone projects + +### Phase 2: Cloud-Specific Fixes (basic-memory-cloud) + +**Priority: P1 - High** + +#### 2.1 Fix Resource Leaks + +```python +# apps/api/src/basic_memory_cloud_api/sync_worker.py + +async def send_keepalive(): + """Send keepalive pings using proper session management""" + # Use context manager to ensure cleanup + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=5) + ) as session: + while True: + try: + await session.get(f"https://{fly_app_name}.fly.dev/health") + await asyncio.sleep(10) + except asyncio.CancelledError: + raise # Exit cleanly + except Exception as e: + logger.warning(f"Keepalive failed: {e}") +``` + +#### 2.2 Improve Circuit Breaker + +Track failures across sync cycles instead of resetting every 30s: + +```python +# Persistent failure tracking +class SyncWorker: + def __init__(self): + self._persistent_failures: Dict[str, int] = {} # file -> failure_count + self._failure_window_start = time.time() + + async def should_skip_file(self, file_path: str) -> bool: + # Skip files that failed >3 times in last hour + if self._persistent_failures.get(file_path, 0) > 3: + if time.time() - self._failure_window_start < 3600: + return True + return False +``` + +### Phase 3: Measurement & Decision + +**Priority: P2 - Future** + +After implementing Phases 1-2, collect metrics for 2 weeks: +- Memory usage per tenant sync +- Sync duration (scan + process) +- Concurrent sync load at peak times +- OOM incidents +- Resource costs + +**UberSync Decision Criteria**: + +Build centralized sync service ONLY if metrics show: +- ✅ Core fixes insufficient for >100 tenants +- ✅ Resource contention causing problems +- ✅ Need for tenant tier prioritization (paid > free) +- ✅ Cost savings justify complexity + +Otherwise, defer UberSync as premature optimization. + +## How to Evaluate + +### Success Metrics (Phase 1) + +**Memory Usage**: +- ✅ Peak memory <500MB for 2,000+ file projects (was 1.5-1.7GB) +- ✅ Memory usage linear with concurrent files (10 max), not total files +- ✅ Large file memory usage: 64KB chunks (not 16MB) + +**Performance**: +- ✅ Initial scan <30 seconds (was 5+ minutes) +- ✅ Full sync <5 minutes for 2,000+ files (was 15+ minutes) +- ✅ Subsequent syncs <10 seconds (only changed files) + +**Stability**: +- ✅ 2,000+ file projects run on 1GB machines +- ✅ Zero OOM kills in production +- ✅ No degradation with binary files (PDFs, images) + +### Success Metrics (Phase 2) + +**Resource Management**: +- ✅ Zero aiohttp session leaks (verified via monitoring) +- ✅ Circuit breaker prevents repeated failures (>3 fails = skip for 1 hour) +- ✅ Tenant syncs distributed over 30s window (no thundering herd) + +**Observability**: +- ✅ Logfire traces show memory usage per sync +- ✅ Clear logging of skipped files and reasons +- ✅ Metrics on sync duration, file counts, failure rates + +### Test Plan + +**Unit Tests** (basic-memory): +- mtime comparison logic +- Streaming checksum correctness +- Semaphore limiting (mock 100 files, verify max 10 concurrent) +- LRU cache eviction +- Checksum computation: streaming vs non-streaming equivalence + +**Integration Tests** (basic-memory): +- Large file handling (create 20MB test file) +- Mixed file types (text + binary) +- Changed file detection via mtime +- Sync with 1,000+ files + +**Load Tests** (basic-memory-cloud): +- Test on tenant-6d2ff1a3 (2,621 files, 31 PDFs) +- Monitor memory during full sync with Logfire +- Measure scan and sync duration +- Run on 1GB machine (downgrade from 2GB to verify) +- Simulate 10 concurrent tenant syncs + +**Regression Tests**: +- Verify existing sync scenarios still work +- CLI sync behavior unchanged +- File watcher integration unaffected + +### Performance Benchmarks + +Establish baseline, then compare after each phase: + +| Metric | Baseline | Phase 1 Target | Phase 2 Target | +|--------|----------|----------------|----------------| +| Peak Memory (2,600 files) | 1.5-1.7GB | <500MB | <450MB | +| Initial Scan Time | 5+ min | <30 sec | <30 sec | +| Full Sync Time | 15+ min | <5 min | <5 min | +| Subsequent Sync | 2+ min | <10 sec | <10 sec | +| OOM Incidents/Week | 2-3 | 0 | 0 | +| Min RAM Required | 2GB | 1GB | 1GB | + +## Implementation Phases + +### Phase 0.5: Database Schema & Streaming Foundation + +**Priority: P0 - Required for Phase 1** + +This phase establishes the foundation for streaming sync with mtime-based change detection. + +**Database Schema Changes**: +- [x] Add `mtime` column to Entity model (REAL type for float timestamp) +- [x] Add `size` column to Entity model (INTEGER type for file size in bytes) +- [x] Create Alembic migration for new columns (nullable initially) +- [x] Add indexes on `(file_path, project_id)` for optimistic upsert performance +- [ ] Backfill existing entities with mtime/size from filesystem + +**Streaming Architecture**: +- [x] Replace `os.walk()` with `os.scandir()` for cached stat info +- [ ] Eliminate `get_db_file_state()` - no upfront SELECT all entities +- [x] Implement streaming iterator `_scan_directory_streaming()` +- [x] Add `get_by_file_path()` optimized query (single file lookup) +- [x] Add `get_all_file_paths()` for deletion detection (paths only, no entities) + +**Benefits**: +- **50% fewer network calls** on Tigris (scandir returns cached stat) +- **No large dicts in memory** (process files one at a time) +- **Indexed lookups** instead of full table scan +- **Foundation for mtime comparison** (Phase 1) + +**Code Changes**: + +```python +# Before: Load all entities upfront +db_paths = await self.get_db_file_state() # SELECT * FROM entity WHERE project_id = ? +scan_result = await self.scan_directory() # os.walk() + stat() per file + +# After: Stream and query incrementally +async for file_path, stat_info in self.scan_directory(): # scandir() with cached stat + db_entity = await self.entity_repository.get_by_file_path(rel_path) # Indexed lookup + # Process immediately, no accumulation +``` + +**Files Modified**: +- `src/basic_memory/models.py` - Add mtime/size columns +- `alembic/versions/xxx_add_mtime_size.py` - Migration +- `src/basic_memory/sync/sync_service.py` - Streaming implementation +- `src/basic_memory/repository/entity_repository.py` - Add get_all_file_paths() + +**Migration Strategy**: +```sql +-- Migration: Add nullable columns +ALTER TABLE entity ADD COLUMN mtime REAL; +ALTER TABLE entity ADD COLUMN size INTEGER; + +-- Backfill from filesystem during first sync after upgrade +-- (Handled in sync_service on first scan) +``` + +### Phase 1: Core Fixes + +**mtime-based scanning**: +- [x] Add mtime/size columns to Entity model (completed in Phase 0.5) +- [x] Database migration (alembic) (completed in Phase 0.5) +- [x] Refactor `scan()` to use streaming architecture with mtime/size comparison +- [x] Update `sync_markdown_file()` and `sync_regular_file()` to store mtime/size in database +- [x] Only compute checksums for changed files (mtime/size differ) +- [x] Unit tests for streaming scan (6 tests passing) +- [ ] Integration test with 1,000 files (defer to benchmarks) + +**Streaming checksums**: +- [x] Implement `_compute_checksum_streaming()` with chunked reading +- [x] Add file size threshold logic (1MB) +- [x] Test with large files (16MB PDF) +- [x] Verify memory usage stays constant +- [x] Test checksum equivalence (streaming vs non-streaming) + +**Bounded concurrency**: +- [x] Add semaphore (10 concurrent) to `_read_file_async()` (already existed) +- [x] Add LRU cache for failures (100 max) (already existed) +- [ ] Review thread pool size configuration +- [ ] Load test with 2,000+ files +- [ ] Verify <500MB peak memory + +**Cleanup & Optimization**: +- [x] Eliminate `get_db_file_state()` - no upfront SELECT all entities (streaming architecture complete) +- [x] Consolidate file operations in FileService (eliminate duplicate checksum logic) +- [x] Add aiofiles dependency (already present) +- [x] FileService streaming checksums for files >1MB +- [x] SyncService delegates all file operations to FileService +- [x] Complete true async I/O refactoring - all file operations use aiofiles + - [x] Added `FileService.read_file_content()` using aiofiles + - [x] Removed `SyncService._read_file_async()` wrapper method + - [x] Removed `SyncService._compute_checksum_async()` wrapper method + - [x] Inlined all 7 checksum calls to use `file_service.compute_checksum()` directly + - [x] All file I/O operations now properly consolidated in FileService with non-blocking I/O +- [x] Removed sync_status_service completely (unnecessary complexity and state tracking) + - [x] Removed `sync_status_service.py` and `sync_status` MCP tool + - [x] Removed all `sync_status_tracker` calls from `sync_service.py` + - [x] Removed migration status checks from MCP tools (`write_note`, `read_note`, `build_context`) + - [x] Removed `check_migration_status()` and `wait_for_migration_or_return_status()` from `utils.py` + - [x] Removed all related tests (4 test files deleted) + - [x] All 1184 tests passing + +**Phase 1 Implementation Summary:** + +Phase 1 is now complete with all core fixes implemented and tested: + +1. **Streaming Architecture** (Phase 0.5 + Phase 1): + - Replaced `os.walk()` with `os.scandir()` for cached stat info + - Eliminated upfront `get_db_file_state()` SELECT query + - Implemented `_scan_directory_streaming()` for incremental processing + - Added indexed `get_by_file_path()` lookups + - Result: 50% fewer network calls on TigrisFS, no large dicts in memory + +2. **mtime-based Change Detection**: + - Added `mtime` and `size` columns to Entity model + - Alembic migration completed and deployed + - Only compute checksums when mtime/size differs from database + - Result: ~90% reduction in checksum operations during typical syncs + +3. **True Async I/O with aiofiles**: + - All file operations consolidated in FileService + - `FileService.compute_checksum()`: 64KB chunked reading for constant memory (lines 261-296 of file_service.py) + - `FileService.read_file_content()`: Non-blocking file reads with aiofiles (lines 160-193 of file_service.py) + - Removed all wrapper methods from SyncService (`_read_file_async`, `_compute_checksum_async`) + - Semaphore controls concurrency (max 10 concurrent file operations) + - Result: Constant memory usage regardless of file size, true non-blocking I/O + +4. **Test Coverage**: + - 41/43 sync tests passing (2 skipped as expected) + - Circuit breaker tests updated for new architecture + - Streaming checksum equivalence verified + - All edge cases covered (large files, concurrent operations, failures) + +**Key Files Modified**: +- `src/basic_memory/models.py` - Added mtime/size columns +- `alembic/versions/xxx_add_mtime_size.py` - Database migration +- `src/basic_memory/sync/sync_service.py` - Streaming implementation, removed wrapper methods +- `src/basic_memory/services/file_service.py` - Added `read_file_content()`, streaming checksums +- `src/basic_memory/repository/entity_repository.py` - Added `get_all_file_paths()` +- `tests/sync/test_sync_service.py` - Updated circuit breaker test mocks + +**Performance Improvements Achieved**: +- Memory usage: Constant per file (64KB chunks) vs full file in memory +- Scan speed: Stat-only scan (no checksums for unchanged files) +- I/O efficiency: True async with aiofiles (no thread pool blocking) +- Network efficiency: 50% fewer calls on TigrisFS via scandir caching +- Architecture: Clean separation of concerns (FileService owns all file I/O) +- Reduced complexity: Removed unnecessary sync_status_service state tracking + +**Observability**: +- [x] Added Logfire instrumentation to `sync_file()` and `sync_markdown_file()` +- [x] Logfire disabled by default via `ignore_no_config = true` in pyproject.toml +- [x] No telemetry in FOSS version unless explicitly configured +- [x] Cloud deployment can enable Logfire for performance monitoring + +**Next Steps**: Phase 1.5 scan watermark optimization for large project performance. + +### Phase 1.5: Scan Watermark Optimization + +**Priority: P0 - Critical for Large Projects** + +This phase addresses Issue #388 where large projects (1,460+ files) take 7+ minutes for sync operations even when no files have changed. + +**Problem Analysis**: + +From production data (tenant-0a20eb58): +- Total sync time: 420-450 seconds (7+ minutes) with 0 changes +- Scan phase: 321 seconds (75% of total time) +- Per-file cost: 220ms × 1,460 files = 5+ minutes +- Root cause: Network I/O to TigrisFS for stat operations (even with mtime columns) +- 15 concurrent syncs every 30 seconds compounds the problem + +**Current Behavior** (Phase 1): +```python +async def scan(self, directory: Path): + """Scan filesystem using mtime/size comparison""" + # Still stats ALL 1,460 files every sync cycle + async for file_path, stat_info in self._scan_directory_streaming(): + db_entity = await self.entity_repository.get_by_file_path(file_path) + # Compare mtime/size, skip unchanged files + # Only checksum if changed (✅ already optimized) +``` + +**Problem**: Even with mtime optimization, we stat every file on every scan. On TigrisFS (network FUSE mount), this means 1,460 network calls taking 5+ minutes. + +**Solution: Scan Watermark + File Count Detection** + +Track when we last scanned and how many files existed. Use filesystem-level filtering to only examine files modified since last scan. + +**Key Insight**: File count changes signal deletions +- Count same → incremental scan (95% of syncs) +- Count increased → new files found by incremental (4% of syncs) +- Count decreased → files deleted, need full scan (1% of syncs) + +**Database Schema Changes**: + +Add to Project model: +```python +last_scan_timestamp: float | None # Unix timestamp of last successful scan start +last_file_count: int | None # Number of files found in last scan +``` + +**Implementation Strategy**: + +```python +async def scan(self, directory: Path): + """Smart scan using watermark and file count""" + project = await self.project_repository.get_current() + + # Step 1: Quick file count (fast on TigrisFS: 1.4s for 1,460 files) + current_count = await self._quick_count_files(directory) + + # Step 2: Determine scan strategy + if project.last_file_count is None: + # First sync ever → full scan + file_paths = await self._scan_directory_full(directory) + scan_type = "full_initial" + + elif current_count < project.last_file_count: + # Files deleted → need full scan to detect which ones + file_paths = await self._scan_directory_full(directory) + scan_type = "full_deletions" + logger.info(f"File count decreased ({project.last_file_count} → {current_count}), running full scan") + + elif project.last_scan_timestamp is not None: + # Incremental scan: only files modified since last scan + file_paths = await self._scan_directory_modified_since( + directory, + project.last_scan_timestamp + ) + scan_type = "incremental" + logger.info(f"Incremental scan since {project.last_scan_timestamp}, found {len(file_paths)} changed files") + else: + # Fallback to full scan + file_paths = await self._scan_directory_full(directory) + scan_type = "full_fallback" + + # Step 3: Process changed files (existing logic) + for file_path in file_paths: + await self._process_file(file_path) + + # Step 4: Update watermark AFTER successful scan + await self.project_repository.update( + project.id, + last_scan_timestamp=time.time(), # Start of THIS scan + last_file_count=current_count + ) + + # Step 5: Record metrics + logfire.metric_counter(f"sync.scan.{scan_type}").add(1) + logfire.metric_histogram("sync.scan.files_scanned", unit="files").record(len(file_paths)) +``` + +**Helper Methods**: + +```python +async def _quick_count_files(self, directory: Path) -> int: + """Fast file count using find command""" + # TigrisFS: 1.4s for 1,460 files + result = await asyncio.create_subprocess_shell( + f'find "{directory}" -type f | wc -l', + stdout=asyncio.subprocess.PIPE + ) + stdout, _ = await result.communicate() + return int(stdout.strip()) + +async def _scan_directory_modified_since( + self, + directory: Path, + since_timestamp: float +) -> List[str]: + """Use find -newermt for filesystem-level filtering""" + # Convert timestamp to find-compatible format + since_date = datetime.fromtimestamp(since_timestamp).strftime("%Y-%m-%d %H:%M:%S") + + # TigrisFS: 0.2s for 0 changed files (vs 5+ minutes for full scan) + result = await asyncio.create_subprocess_shell( + f'find "{directory}" -type f -newermt "{since_date}"', + stdout=asyncio.subprocess.PIPE + ) + stdout, _ = await result.communicate() + + # Convert absolute paths to relative + file_paths = [] + for line in stdout.decode().splitlines(): + if line: + rel_path = Path(line).relative_to(directory).as_posix() + file_paths.append(rel_path) + + return file_paths +``` + +**TigrisFS Testing Results** (SSH to production-basic-memory-tenant-0a20eb58): + +```bash +# Full file count +$ time find . -type f | wc -l +1460 +real 0m1.362s # ✅ Acceptable + +# Incremental scan (1 hour window) +$ time find . -type f -newermt "2025-01-20 10:00:00" | wc -l +0 +real 0m0.161s # ✅ 8.5x faster! + +# Incremental scan (24 hours) +$ time find . -type f -newermt "2025-01-19 11:00:00" | wc -l +0 +real 0m0.239s # ✅ 5.7x faster! +``` + +**Conclusion**: `find -newermt` works perfectly on TigrisFS and provides massive speedup. + +**Expected Performance Improvements**: + +| Scenario | Files Changed | Current Time | With Watermark | Speedup | +|----------|---------------|--------------|----------------|---------| +| No changes (common) | 0 | 420s | ~2s | 210x | +| Few changes | 5-10 | 420s | ~5s | 84x | +| Many changes | 100+ | 420s | ~30s | 14x | +| Deletions (rare) | N/A | 420s | 420s | 1x | + +**Full sync breakdown** (1,460 files, 0 changes): +- File count: 1.4s +- Incremental scan: 0.2s +- Database updates: 0.4s +- **Total: ~2s (225x faster)** + +**Metrics to Track**: + +```python +# Scan type distribution +logfire.metric_counter("sync.scan.full_initial").add(1) +logfire.metric_counter("sync.scan.full_deletions").add(1) +logfire.metric_counter("sync.scan.incremental").add(1) + +# Performance metrics +logfire.metric_histogram("sync.scan.duration", unit="ms").record(scan_ms) +logfire.metric_histogram("sync.scan.files_scanned", unit="files").record(file_count) +logfire.metric_histogram("sync.scan.files_changed", unit="files").record(changed_count) + +# Watermark effectiveness +logfire.metric_histogram("sync.scan.watermark_age", unit="s").record( + time.time() - project.last_scan_timestamp +) +``` + +**Edge Cases Handled**: + +1. **First sync**: No watermark → full scan (expected) +2. **Deletions**: File count decreased → full scan (rare but correct) +3. **Clock skew**: Use scan start time, not end time (captures files created during scan) +4. **Scan failure**: Don't update watermark on failure (retry will re-scan) +5. **New files**: Count increased → incremental scan finds them (common, fast) + +**Files to Modify**: +- `src/basic_memory/models.py` - Add last_scan_timestamp, last_file_count to Project +- `alembic/versions/xxx_add_scan_watermark.py` - Migration for new columns +- `src/basic_memory/sync/sync_service.py` - Implement watermark logic +- `src/basic_memory/repository/project_repository.py` - Update methods +- `tests/sync/test_sync_watermark.py` - Test watermark behavior + +**Test Plan**: +- [x] SSH test on TigrisFS confirms `find -newermt` works (completed) +- [x] Unit tests for scan strategy selection (4 tests) +- [x] Unit tests for file count detection (integrated in strategy tests) +- [x] Integration test: verify incremental scan finds changed files (4 tests) +- [x] Integration test: verify deletion detection triggers full scan (2 tests) +- [ ] Load test on tenant-0a20eb58 (1,460 files) - pending production deployment +- [ ] Verify <3s for no-change sync - pending production deployment + +**Implementation Status**: ✅ **COMPLETED** + +**Code Changes** (Commit: `fb16055d`): +- ✅ Added `last_scan_timestamp` and `last_file_count` to Project model +- ✅ Created database migration `e7e1f4367280_add_scan_watermark_tracking_to_project.py` +- ✅ Implemented smart scan strategy selection in `sync_service.py` +- ✅ Added `_quick_count_files()` using `find | wc -l` (~1.4s for 1,460 files) +- ✅ Added `_scan_directory_modified_since()` using `find -newermt` (~0.2s) +- ✅ Added `_scan_directory_full()` wrapper for full scans +- ✅ Watermark update logic after successful sync (uses sync START time) +- ✅ Logfire metrics for scan types and performance tracking + +**Test Coverage** (18 tests in `test_sync_service_incremental.py`): +- ✅ Scan strategy selection (4 tests) + - First sync uses full scan + - File count decreased triggers full scan + - Same file count uses incremental scan + - Increased file count uses incremental scan +- ✅ Incremental scan base cases (4 tests) + - No changes scenario + - Detects new files + - Detects modified files + - Detects multiple changes +- ✅ Deletion detection (2 tests) + - Single file deletion + - Multiple file deletions +- ✅ Move detection (2 tests) + - Moves require full scan (renames don't update mtime) + - Moves detected in full scan via checksum +- ✅ Watermark update (3 tests) + - Watermark updated after successful sync + - Watermark uses sync start time + - File count accuracy +- ✅ Edge cases (3 tests) + - Concurrent file changes + - Empty directory handling + - Respects .gitignore patterns + +**Performance Expectations** (to be verified in production): +- No changes: 420s → ~2s (210x faster) +- Few changes (5-10): 420s → ~5s (84x faster) +- Many changes (100+): 420s → ~30s (14x faster) +- Deletions: 420s → 420s (full scan, rare case) + +**Rollout Strategy**: +1. ✅ Code complete and tested (18 new tests, all passing) +2. ✅ Pushed to `phase-0.5-streaming-foundation` branch +3. ⏳ Windows CI tests running +4. 📊 Deploy to staging tenant with watermark optimization +5. 📊 Monitor scan performance metrics via Logfire +6. 📊 Verify no missed files (compare full vs incremental results) +7. 📊 Deploy to production tenant-0a20eb58 +8. 📊 Measure actual improvement (expect 420s → 2-3s) + +**Success Criteria**: +- ✅ Implementation complete with comprehensive tests +- [ ] No-change syncs complete in <3 seconds (was 420s) - pending production test +- [ ] Incremental scans (95% of cases) use watermark - pending production test +- [ ] Deletion detection works correctly (full scan when needed) - tested in unit tests ✅ +- [ ] No files missed due to watermark logic - tested in unit tests ✅ +- [ ] Metrics show scan type distribution matches expectations - pending production test + +**Next Steps**: +1. Production deployment to tenant-0a20eb58 +2. Measure actual performance improvements +3. Monitor metrics for 1 week +4. Phase 2 cloud-specific fixes +5. Phase 3 production measurement and UberSync decision + +### Phase 2: Cloud Fixes + +**Resource leaks**: +- [ ] Fix aiohttp session context manager +- [ ] Implement persistent circuit breaker +- [ ] Add memory monitoring/alerts +- [ ] Test on production tenant + +**Sync coordination**: +- [ ] Implement hash-based staggering +- [ ] Add jitter to sync intervals +- [ ] Load test with 10 concurrent tenants +- [ ] Verify no thundering herd + +### Phase 3: Measurement + +**Deploy to production**: +- [ ] Deploy Phase 1+2 changes +- [ ] Downgrade tenant-6d2ff1a3 to 1GB +- [ ] Monitor for OOM incidents + +**Collect metrics**: +- [ ] Memory usage patterns +- [ ] Sync duration distributions +- [ ] Concurrent sync load +- [ ] Cost analysis + +**UberSync decision**: +- [ ] Review metrics against decision criteria +- [ ] Document findings +- [ ] Create SPEC-18 for UberSync if needed + +## Related Issues + +### basic-memory (core) +- [#383](https://github.com/basicmachines-co/basic-memory/issues/383) - Refactor sync to use mtime-based scanning +- [#382](https://github.com/basicmachines-co/basic-memory/issues/382) - Optimize memory for large file syncs +- [#371](https://github.com/basicmachines-co/basic-memory/issues/371) - aiofiles for non-blocking I/O (future) + +### basic-memory-cloud +- [#198](https://github.com/basicmachines-co/basic-memory-cloud/issues/198) - Memory optimization for sync worker +- [#189](https://github.com/basicmachines-co/basic-memory-cloud/issues/189) - Circuit breaker for infinite retry loops + +## References + +**Standard sync tools using mtime**: +- rsync: Uses mtime-based comparison by default, only checksums on `--checksum` flag +- rclone: Default is mtime/size, `--checksum` mode optional +- syncthing: Block-level sync with mtime tracking + +**fsnotify polling** (future consideration): +- [fsnotify/fsnotify#9](https://github.com/fsnotify/fsnotify/issues/9) - Polling mode for network filesystems + +## Notes + +### Why Not UberSync Now? + +**Premature Optimization**: +- Current problems are algorithmic, not architectural +- No evidence that multi-tenant coordination is the issue +- Single tenant OOM proves algorithm is the problem + +**Benefits of Core-First Approach**: +- ✅ Helps all users (CLI + Cloud) +- ✅ Lower risk (no new service) +- ✅ Clear path (issues specify fixes) +- ✅ Can defer UberSync until proven necessary + +**When UberSync Makes Sense**: +- >100 active tenants causing resource contention +- Need for tenant tier prioritization (paid > free) +- Centralized observability requirements +- Cost optimization at scale + +### Migration Strategy + +**Backward Compatibility**: +- New mtime/size columns nullable initially +- Existing entities sync normally (compute mtime on first scan) +- No breaking changes to MCP API +- CLI behavior unchanged + +**Rollout**: +1. Deploy to staging with test tenant +2. Validate memory/performance improvements +3. Deploy to production (blue-green) +4. Monitor for 1 week +5. Downgrade tenant machines if successful + +## Further Considerations + +### Version Control System (VCS) Integration + +**Context:** Users frequently request git versioning, and large projects with PDFs/images pose memory challenges. + +#### Git-Based Sync + +**Approach:** Use git for change detection instead of custom mtime comparison. + +```python +# Git automatically tracks changes +repo = git.Repo(project_path) +repo.git.add(A=True) +diff = repo.index.diff('HEAD') + +for change in diff: + if change.change_type == 'M': # Modified + await sync_file(change.b_path) +``` + +**Pros:** +- ✅ Proven, battle-tested change detection +- ✅ Built-in rename/move detection (similarity index) +- ✅ Efficient for cloud sync (git protocol over HTTP) +- ✅ Could enable version history as bonus feature +- ✅ Users want git integration anyway + +**Cons:** +- ❌ User confusion (`.git` folder in knowledge base) +- ❌ Conflicts with existing git repos (submodule complexity) +- ❌ Adds dependency (git binary or dulwich/pygit2) +- ❌ Less control over sync logic +- ❌ Doesn't solve large file problem (PDFs still checksummed) +- ❌ Git LFS adds complexity + +#### Jujutsu (jj) Alternative + +**Why jj is compelling:** + +1. **Working Copy as Source of Truth** + - Git: Staging area is intermediate state + - Jujutsu: Working copy IS a commit + - Aligns with "files are source of truth" philosophy! + +2. **Automatic Change Tracking** + - No manual staging required + - Working copy changes tracked automatically + - Better fit for sync operations vs git's commit-centric model + +3. **Conflict Handling** + - User edits + sync changes both preserved + - Operation log vs linear history + - Built for operations, not just history + +**Cons:** +- ❌ New/immature (2020 vs git's 2005) +- ❌ Not universally available +- ❌ Steeper learning curve for users +- ❌ No LFS equivalent yet +- ❌ Still doesn't solve large file checksumming + +#### Git Index Format (Hybrid Approach) + +**Best of both worlds:** Use git's index format without full git repo. + +```python +from dulwich.index import Index # Pure Python + +# Use git index format for tracking +idx = Index(project_path / '.basic-memory' / 'index') + +for file in files: + stat = file.stat() + if idx.get(file) and idx[file].mtime == stat.st_mtime: + continue # Unchanged (git's proven logic) + + await sync_file(file) + idx[file] = (stat.st_mtime, stat.st_size, sha) +``` + +**Pros:** +- ✅ Git's proven change detection logic +- ✅ No user-visible `.git` folder +- ✅ No git dependency (pure Python) +- ✅ Full control over sync + +**Cons:** +- ❌ Adds dependency (dulwich) +- ❌ Doesn't solve large files +- ❌ No built-in versioning + +### Large File Handling + +**Problem:** PDFs/images cause memory issues regardless of VCS choice. + +**Solutions (Phase 1+):** + +**1. Skip Checksums for Large Files** +```python +if stat.st_size > 10_000_000: # 10MB threshold + checksum = None # Use mtime/size only + logger.info(f"Skipping checksum for {file_path}") +``` + +**2. Partial Hashing** +```python +if file.suffix in ['.pdf', '.jpg', '.png']: + # Hash first/last 64KB instead of entire file + checksum = hash_partial(file, chunk_size=65536) +``` + +**3. External Blob Storage** +```python +if stat.st_size > 10_000_000: + blob_id = await upload_to_tigris_blob(file) + entity.blob_id = blob_id + entity.file_path = None # Not in main sync +``` + +### Recommendation & Timeline + +**Phase 0.5-1 (Now):** Custom streaming + mtime +- ✅ Solves urgent memory issues +- ✅ No dependencies +- ✅ Full control +- ✅ Skip checksums for large files (>10MB) +- ✅ Proven pattern (rsync/rclone) + +**Phase 2 (After metrics):** Git index format exploration +```python +# Optional: Use git index for tracking if beneficial +from dulwich.index import Index +# No git repo, just index file format +``` + +**Future (User feature):** User-facing versioning +```python +# Let users opt into VCS: +basic-memory config set versioning git +basic-memory config set versioning jj +basic-memory config set versioning none # Current behavior + +# Integrate with their chosen workflow +# Not forced upon them +``` + +**Rationale:** +1. **Don't block on VCS decision** - Memory issues are P0 +2. **Learn from deployment** - See actual usage patterns +3. **Keep options open** - Can add git/jj later +4. **Files as source of truth** - Core philosophy preserved +5. **Large files need attention regardless** - VCS won't solve that + +**Decision Point:** +- If Phase 0.5/1 achieves memory targets → VCS integration deferred +- If users strongly request versioning → Add as opt-in feature +- If change detection becomes bottleneck → Explore git index format + +## Agent Assignment + +**Phase 1 Implementation**: `python-developer` agent +- Expertise in FastAPI, async Python, database migrations +- Handles basic-memory core changes + +**Phase 2 Implementation**: `python-developer` agent +- Same agent continues with cloud-specific fixes +- Maintains consistency across phases + +**Phase 3 Review**: `system-architect` agent +- Analyzes metrics and makes UberSync decision +- Creates SPEC-18 if centralized service needed diff --git a/src/basic_memory/alembic/versions/9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py b/src/basic_memory/alembic/versions/9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py new file mode 100644 index 00000000..386a9e41 --- /dev/null +++ b/src/basic_memory/alembic/versions/9d9c1cb7d8f5_add_mtime_and_size_columns_to_entity_.py @@ -0,0 +1,49 @@ +"""Add mtime and size columns to Entity for sync optimization + +Revision ID: 9d9c1cb7d8f5 +Revises: a1b2c3d4e5f6 +Create Date: 2025-10-20 05:07:55.173849 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "9d9c1cb7d8f5" +down_revision: Union[str, None] = "a1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("entity", schema=None) as batch_op: + batch_op.add_column(sa.Column("mtime", sa.Float(), nullable=True)) + batch_op.add_column(sa.Column("size", sa.Integer(), nullable=True)) + batch_op.drop_constraint(batch_op.f("fk_entity_project_id"), type_="foreignkey") + batch_op.create_foreign_key( + batch_op.f("fk_entity_project_id"), "project", ["project_id"], ["id"] + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("entity", schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f("fk_entity_project_id"), type_="foreignkey") + batch_op.create_foreign_key( + batch_op.f("fk_entity_project_id"), + "project", + ["project_id"], + ["id"], + ondelete="CASCADE", + ) + batch_op.drop_column("size") + batch_op.drop_column("mtime") + + # ### end Alembic commands ### diff --git a/src/basic_memory/alembic/versions/e7e1f4367280_add_scan_watermark_tracking_to_project.py b/src/basic_memory/alembic/versions/e7e1f4367280_add_scan_watermark_tracking_to_project.py new file mode 100644 index 00000000..529d79ed --- /dev/null +++ b/src/basic_memory/alembic/versions/e7e1f4367280_add_scan_watermark_tracking_to_project.py @@ -0,0 +1,37 @@ +"""Add scan watermark tracking to Project + +Revision ID: e7e1f4367280 +Revises: 9d9c1cb7d8f5 +Create Date: 2025-10-20 16:42:46.625075 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "e7e1f4367280" +down_revision: Union[str, None] = "9d9c1cb7d8f5" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project", schema=None) as batch_op: + batch_op.add_column(sa.Column("last_scan_timestamp", sa.Float(), nullable=True)) + batch_op.add_column(sa.Column("last_file_count", sa.Integer(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("project", schema=None) as batch_op: + batch_op.drop_column("last_file_count") + batch_op.drop_column("last_scan_timestamp") + + # ### end Alembic commands ### diff --git a/src/basic_memory/deps.py b/src/basic_memory/deps.py index 11ce5ddb..35b73e08 100644 --- a/src/basic_memory/deps.py +++ b/src/basic_memory/deps.py @@ -321,6 +321,7 @@ async def get_sync_service( entity_parser: EntityParserDep, entity_repository: EntityRepositoryDep, relation_repository: RelationRepositoryDep, + project_repository: ProjectRepositoryDep, search_service: SearchServiceDep, file_service: FileServiceDep, ) -> SyncService: # pragma: no cover @@ -334,6 +335,7 @@ async def get_sync_service( entity_parser=entity_parser, entity_repository=entity_repository, relation_repository=relation_repository, + project_repository=project_repository, search_service=search_service, file_service=file_service, ) diff --git a/src/basic_memory/file_utils.py b/src/basic_memory/file_utils.py index fa5c0678..fe131282 100644 --- a/src/basic_memory/file_utils.py +++ b/src/basic_memory/file_utils.py @@ -5,6 +5,7 @@ from pathlib import Path import re from typing import Any, Dict, Union +import aiofiles import yaml import frontmatter from loguru import logger @@ -52,29 +53,12 @@ async def compute_checksum(content: Union[str, bytes]) -> str: raise FileError(f"Failed to compute checksum: {e}") -async def ensure_directory(path: FilePath) -> None: - """ - Ensure directory exists, creating if necessary. - - Args: - path: Directory path to ensure (Path or string) - - Raises: - FileWriteError: If directory creation fails - """ - try: - # Convert string to Path if needed - path_obj = Path(path) if isinstance(path, str) else path - path_obj.mkdir(parents=True, exist_ok=True) - except Exception as e: # pragma: no cover - logger.error("Failed to create directory", path=str(path), error=str(e)) - raise FileWriteError(f"Failed to create directory {path}: {e}") - - async def write_file_atomic(path: FilePath, content: str) -> None: """ Write file with atomic operation using temporary file. + Uses aiofiles for true async I/O (non-blocking). + Args: path: Target file path (Path or string) content: Content to write @@ -87,7 +71,11 @@ async def write_file_atomic(path: FilePath, content: str) -> None: temp_path = path_obj.with_suffix(".tmp") try: - temp_path.write_text(content, encoding="utf-8") + # Use aiofiles for non-blocking write + async with aiofiles.open(temp_path, mode="w", encoding="utf-8") as f: + await f.write(content) + + # Atomic rename (this is fast, doesn't need async) temp_path.replace(path_obj) logger.debug("Wrote file atomically", path=str(path_obj), content_length=len(content)) except Exception as e: # pragma: no cover @@ -185,69 +173,6 @@ def remove_frontmatter(content: str) -> str: return parts[2].strip() -async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str: - """Update frontmatter fields in a file while preserving all content. - - Only modifies the frontmatter section, leaving all content untouched. - Creates frontmatter section if none exists. - Returns checksum of updated file. - - Args: - path: Path to markdown file (Path or string) - updates: Dict of frontmatter fields to update - - Returns: - Checksum of updated file - - Raises: - FileError: If file operations fail - ParseError: If frontmatter parsing fails - """ - try: - # Convert string to Path if needed - path_obj = Path(path) if isinstance(path, str) else path - - # Read current content - content = path_obj.read_text(encoding="utf-8") - - # Parse current frontmatter with proper error handling for malformed YAML - current_fm = {} - if has_frontmatter(content): - try: - current_fm = parse_frontmatter(content) - content = remove_frontmatter(content) - except (ParseError, yaml.YAMLError) as e: - # Log warning and treat as plain markdown without frontmatter - logger.warning( - f"Failed to parse YAML frontmatter in {path_obj}: {e}. " - "Treating file as plain markdown without frontmatter." - ) - # Keep full content, treat as having no frontmatter - current_fm = {} - - # Update frontmatter - new_fm = {**current_fm, **updates} - - # Write new file with updated frontmatter - yaml_fm = yaml.dump(new_fm, sort_keys=False, allow_unicode=True) - final_content = f"---\n{yaml_fm}---\n\n{content.strip()}" - - logger.debug("Updating frontmatter", path=str(path_obj), update_keys=list(updates.keys())) - - await write_file_atomic(path_obj, final_content) - return await compute_checksum(final_content) - - except Exception as e: # pragma: no cover - # Only log real errors (not YAML parsing, which is handled above) - if not isinstance(e, (ParseError, yaml.YAMLError)): - logger.error( - "Failed to update frontmatter", - path=str(path) if isinstance(path, (str, Path)) else "", - error=str(e), - ) - raise FileError(f"Failed to update frontmatter: {e}") - - def dump_frontmatter(post: frontmatter.Post) -> str: """ Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format. diff --git a/src/basic_memory/mcp/tools/__init__.py b/src/basic_memory/mcp/tools/__init__.py index ea3a6e33..f7844529 100644 --- a/src/basic_memory/mcp/tools/__init__.py +++ b/src/basic_memory/mcp/tools/__init__.py @@ -18,7 +18,6 @@ from basic_memory.mcp.tools.canvas import canvas from basic_memory.mcp.tools.list_directory import list_directory from basic_memory.mcp.tools.edit_note import edit_note from basic_memory.mcp.tools.move_note import move_note -from basic_memory.mcp.tools.sync_status import sync_status from basic_memory.mcp.tools.project_management import ( list_memory_projects, create_memory_project, @@ -44,7 +43,6 @@ __all__ = [ "recent_activity", "search", "search_notes", - "sync_status", "view_note", "write_note", ] diff --git a/src/basic_memory/mcp/tools/build_context.py b/src/basic_memory/mcp/tools/build_context.py index a8c79718..68d399df 100644 --- a/src/basic_memory/mcp/tools/build_context.py +++ b/src/basic_memory/mcp/tools/build_context.py @@ -106,28 +106,6 @@ async def build_context( # Get the active project using the new stateless approach active_project = await get_active_project(client, project, context) - # Check migration status and wait briefly if needed - from basic_memory.mcp.tools.utils import wait_for_migration_or_return_status - - migration_status = await wait_for_migration_or_return_status( - timeout=5.0, project_name=active_project.name - ) - if migration_status: # pragma: no cover - # Return a proper GraphContext with status message - from basic_memory.schemas.memory import MemoryMetadata - from datetime import datetime - - return GraphContext( - results=[], - metadata=MemoryMetadata( - depth=depth or 1, - timeframe=timeframe, - generated_at=datetime.now().astimezone(), - primary_count=0, - related_count=0, - uri=migration_status, # Include status in metadata - ), - ) project_url = active_project.project_url response = await call_get( diff --git a/src/basic_memory/mcp/tools/read_note.py b/src/basic_memory/mcp/tools/read_note.py index 29dc2996..78b4a101 100644 --- a/src/basic_memory/mcp/tools/read_note.py +++ b/src/basic_memory/mcp/tools/read_note.py @@ -97,14 +97,6 @@ async def read_note( ) return f"# Error\n\nIdentifier '{identifier}' is not allowed - paths must stay within project boundaries" - # Check migration status and wait briefly if needed - from basic_memory.mcp.tools.utils import wait_for_migration_or_return_status - - migration_status = await wait_for_migration_or_return_status( - timeout=5.0, project_name=active_project.name - ) - if migration_status: # pragma: no cover - return f"# System Status\n\n{migration_status}\n\nPlease wait for migration to complete before reading notes." project_url = active_project.project_url # Get the file via REST API - first try direct permalink lookup diff --git a/src/basic_memory/mcp/tools/sync_status.py b/src/basic_memory/mcp/tools/sync_status.py deleted file mode 100644 index c4162b61..00000000 --- a/src/basic_memory/mcp/tools/sync_status.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Sync status tool for Basic Memory MCP server.""" - -from typing import Optional - -from loguru import logger -from fastmcp import Context - -from basic_memory.config import ConfigManager -from basic_memory.mcp.async_client import get_client -from basic_memory.mcp.server import mcp -from basic_memory.mcp.project_context import get_active_project -from basic_memory.services.sync_status_service import sync_status_tracker - - -def _get_all_projects_status() -> list[str]: - """Get status lines for all configured projects.""" - status_lines = [] - - try: - app_config = ConfigManager().config - - if app_config.projects: - status_lines.extend(["", "---", "", "**All Projects Status:**"]) - - for project_name, project_path in app_config.projects.items(): - # Check if this project has sync status - project_sync_status = sync_status_tracker.get_project_status(project_name) - - if project_sync_status: - # Project has tracked sync activity - if project_sync_status.status.value == "watching": - # Project is actively watching for changes (steady state) - status_icon = "👁️" - status_text = "Watching for changes" - elif project_sync_status.status.value == "completed": - # Sync completed but not yet watching - transitional state - status_icon = "✅" - status_text = "Sync completed" - elif project_sync_status.status.value in ["scanning", "syncing"]: - status_icon = "🔄" - status_text = "Sync in progress" - if project_sync_status.files_total > 0: - progress_pct = ( - project_sync_status.files_processed - / project_sync_status.files_total - ) * 100 - status_text += f" ({project_sync_status.files_processed}/{project_sync_status.files_total}, {progress_pct:.0f}%)" - elif project_sync_status.status.value == "failed": - status_icon = "❌" - status_text = f"Sync error: {project_sync_status.error or 'Unknown error'}" - else: - status_icon = "⏸️" - status_text = project_sync_status.status.value.title() - else: - # Project has no tracked sync activity - will be synced automatically - status_icon = "⏳" - status_text = "Pending sync" - - status_lines.append(f"- {status_icon} **{project_name}**: {status_text}") - - except Exception as e: - logger.debug(f"Could not get project config for comprehensive status: {e}") - - return status_lines - - -@mcp.tool( - description="""Check the status of file synchronization and background operations. - - Use this tool to: - - Check if file sync is in progress or completed - - Get detailed sync progress information - - Understand if your files are fully indexed - - Get specific error details if sync operations failed - - Monitor initial project setup and legacy migration - - This covers all sync operations including: - - Initial project setup and file indexing - - Legacy project migration to unified database - - Ongoing file monitoring and updates - - Background processing of knowledge graphs - """, -) -async def sync_status(project: Optional[str] = None, context: Context | None = None) -> str: - """Get current sync status and system readiness information. - - This tool provides detailed information about any ongoing or completed - sync operations, helping users understand when their files are ready. - - Args: - project: Optional project name to get project-specific context - - Returns: - Formatted sync status with progress, readiness, and guidance - """ - logger.info("MCP tool call tool=sync_status") - - async with get_client() as client: - status_lines = [] - - try: - from basic_memory.services.sync_status_service import sync_status_tracker - - # Get overall summary - summary = sync_status_tracker.get_summary() - is_ready = sync_status_tracker.is_ready - - # Header - status_lines.extend( - [ - "# Basic Memory Sync Status", - "", - f"**Current Status**: {summary}", - f"**System Ready**: {'✅ Yes' if is_ready else '🔄 Processing'}", - "", - ] - ) - - if is_ready: - status_lines.extend( - [ - "✅ **All sync operations completed**", - "", - "- File indexing is complete", - "- Knowledge graphs are up to date", - "- All Basic Memory tools are fully operational", - "", - "Your knowledge base is ready for use!", - ] - ) - - # Show all projects status even when ready - status_lines.extend(_get_all_projects_status()) - else: - # System is still processing - show both active and all projects - all_sync_projects = sync_status_tracker.get_all_projects() - - active_projects = [ - p - for p in all_sync_projects.values() - if p.status.value in ["scanning", "syncing"] - ] - failed_projects = [ - p for p in all_sync_projects.values() if p.status.value == "failed" - ] - - if active_projects: - status_lines.extend( - [ - "🔄 **File synchronization in progress**", - "", - "Basic Memory is automatically processing all configured projects and building knowledge graphs.", - "This typically takes 1-3 minutes depending on the amount of content.", - "", - "**Currently Processing:**", - ] - ) - - for project_status in active_projects: - progress = "" - if project_status.files_total > 0: - progress_pct = ( - project_status.files_processed / project_status.files_total - ) * 100 - progress = f" ({project_status.files_processed}/{project_status.files_total}, {progress_pct:.0f}%)" - - status_lines.append( - f"- **{project_status.project_name}**: {project_status.message}{progress}" - ) - - status_lines.extend( - [ - "", - "**What's happening:**", - "- Scanning and indexing markdown files", - "- Building entity and relationship graphs", - "- Settings up full-text search indexes", - "- Processing file changes and updates", - "", - "**What you can do:**", - "- Wait for automatic processing to complete - no action needed", - "- Use this tool again to check progress", - "- Simple operations may work already", - "- All projects will be available once sync finishes", - ] - ) - - # Handle failed projects (independent of active projects) - if failed_projects: - status_lines.extend(["", "❌ **Some projects failed to sync:**", ""]) - - for project_status in failed_projects: - status_lines.append( - f"- **{project_status.project_name}**: {project_status.error or 'Unknown error'}" - ) - - status_lines.extend( - [ - "", - "**Next steps:**", - "1. Check the logs for detailed error information", - "2. Ensure file permissions allow read/write access", - "3. Try restarting the MCP server", - "4. If issues persist, consider filing a support issue", - ] - ) - elif not active_projects: - # No active or failed projects - must be pending - status_lines.extend( - [ - "⏳ **Sync operations pending**", - "", - "File synchronization has been queued but hasn't started yet.", - "This usually resolves automatically within a few seconds.", - ] - ) - - # Add comprehensive project status for all configured projects - all_projects_status = _get_all_projects_status() - if all_projects_status: - status_lines.extend(all_projects_status) - - # Add explanation about automatic syncing if there are unsynced projects - unsynced_count = sum(1 for line in all_projects_status if "⏳" in line) - if unsynced_count > 0 and not is_ready: - status_lines.extend( - [ - "", - "**Note**: All configured projects will be automatically synced during startup.", - ] - ) - - # Add project context if provided - if project: - try: - active_project = await get_active_project(client, project, context) - status_lines.extend( - [ - "", - "---", - "", - f"**Active Project**: {active_project.name}", - f"**Project Path**: {active_project.home}", - ] - ) - except Exception as e: - logger.debug(f"Could not get project info: {e}") - - return "\n".join(status_lines) - - except Exception as e: - return f"""# Sync Status - Error - -❌ **Unable to check sync status**: {str(e)} - -**Troubleshooting:** -- The system may still be starting up -- Try waiting a few seconds and checking again -- Check logs for detailed error information -- Consider restarting if the issue persists -""" diff --git a/src/basic_memory/mcp/tools/utils.py b/src/basic_memory/mcp/tools/utils.py index f2257643..6f4184ef 100644 --- a/src/basic_memory/mcp/tools/utils.py +++ b/src/basic_memory/mcp/tools/utils.py @@ -510,73 +510,3 @@ async def call_delete( except HTTPStatusError as e: raise ToolError(error_message) from e - - -def check_migration_status() -> Optional[str]: - """Check if sync/migration is in progress and return status message if so. - - Returns: - Status message if sync is in progress, None if system is ready - """ - try: - from basic_memory.services.sync_status_service import sync_status_tracker - - if not sync_status_tracker.is_ready: - return sync_status_tracker.get_summary() - return None - except Exception: - # If there's any error checking sync status, assume ready - return None - - -async def wait_for_migration_or_return_status( - timeout: float = 5.0, project_name: Optional[str] = None -) -> Optional[str]: - """Wait briefly for sync/migration to complete, or return status message. - - Args: - timeout: Maximum time to wait for sync completion - project_name: Optional project name to check specific project status. - If provided, only checks that project's readiness. - If None, uses global status check (legacy behavior). - - Returns: - Status message if sync is still in progress, None if ready - """ - try: - from basic_memory.services.sync_status_service import sync_status_tracker - import asyncio - - # Check if we should use project-specific or global status - def is_ready() -> bool: - if project_name: - return sync_status_tracker.is_project_ready(project_name) - return sync_status_tracker.is_ready - - if is_ready(): - return None - - # Wait briefly for sync to complete - start_time = asyncio.get_event_loop().time() - while (asyncio.get_event_loop().time() - start_time) < timeout: - if is_ready(): - return None - await asyncio.sleep(0.1) # Check every 100ms - - # Still not ready after timeout - if project_name: - # For project-specific checks, get project status details - project_status = sync_status_tracker.get_project_status(project_name) - if project_status and project_status.status.value == "failed": - error_msg = project_status.error or "Unknown sync error" - return f"❌ Sync failed for project '{project_name}': {error_msg}" - elif project_status: - return f"🔄 Project '{project_name}' is still syncing: {project_status.message}" - else: - return f"⚠️ Project '{project_name}' status unknown" - else: - # Fall back to global summary for legacy calls - return sync_status_tracker.get_summary() - except Exception: # pragma: no cover - # If there's any error, assume ready - return None diff --git a/src/basic_memory/mcp/tools/write_note.py b/src/basic_memory/mcp/tools/write_note.py index ef07e202..ac587022 100644 --- a/src/basic_memory/mcp/tools/write_note.py +++ b/src/basic_memory/mcp/tools/write_note.py @@ -140,15 +140,6 @@ async def write_note( ) return f"# Error\n\nFolder path '{folder}' is not allowed - paths must stay within project boundaries" - # Check migration status and wait briefly if needed - from basic_memory.mcp.tools.utils import wait_for_migration_or_return_status - - migration_status = await wait_for_migration_or_return_status( - timeout=5.0, project_name=active_project.name - ) - if migration_status: # pragma: no cover - return f"# System Status\n\n{migration_status}\n\nPlease wait for migration to complete before creating notes." - # Process tags using the helper function tag_list = parse_tags(tags) # Create the entity request diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index c8edbc01..4c98a610 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -13,6 +13,7 @@ from sqlalchemy import ( DateTime, Index, JSON, + Float, text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -73,6 +74,12 @@ class Entity(Base): # checksum of file checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True) + # File metadata for sync + # mtime: file modification timestamp (Unix epoch float) for change detection + mtime: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + # size: file size in bytes for quick change detection + size: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + # Metadata and tracking created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now().astimezone() diff --git a/src/basic_memory/models/project.py b/src/basic_memory/models/project.py index 604e808f..373eae6e 100644 --- a/src/basic_memory/models/project.py +++ b/src/basic_memory/models/project.py @@ -9,6 +9,7 @@ from sqlalchemy import ( Text, Boolean, DateTime, + Float, Index, event, ) @@ -61,6 +62,10 @@ class Project(Base): onupdate=lambda: datetime.now(UTC), ) + # Sync optimization - scan watermark tracking + last_scan_timestamp: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + last_file_count: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + # Define relationships to entities, observations, and relations # These relationships will be established once we add project_id to those models entities = relationship("Entity", back_populates="project", cascade="all, delete-orphan") diff --git a/src/basic_memory/repository/entity_repository.py b/src/basic_memory/repository/entity_repository.py index ac216cd2..8f314c52 100644 --- a/src/basic_memory/repository/entity_repository.py +++ b/src/basic_memory/repository/entity_repository.py @@ -63,6 +63,23 @@ class EntityRepository(Repository[Entity]): ) return await self.find_one(query) + async def find_by_checksum(self, checksum: str) -> Sequence[Entity]: + """Find entities with the given checksum. + + Used for move detection - finds entities that may have been moved to a new path. + Multiple entities may have the same checksum if files were copied. + + Args: + checksum: File content checksum to search for + + Returns: + Sequence of entities with matching checksum (may be empty) + """ + query = self.select().where(Entity.checksum == checksum) + # Don't load relationships for move detection - we only need file_path and checksum + result = await self.execute_query(query, use_query_options=False) + return list(result.scalars().all()) + async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool: """Delete entity with the provided file_path. @@ -197,6 +214,21 @@ class EntityRepository(Repository[Entity]): entity = await self._handle_permalink_conflict(entity, session) return entity + async def get_all_file_paths(self) -> List[str]: + """Get all file paths for this project - optimized for deletion detection. + + Returns only file_path strings without loading entities or relationships. + Used by streaming sync to detect deleted files efficiently. + + Returns: + List of file_path strings for all entities in the project + """ + query = select(Entity.file_path) + query = self._add_project_filter(query) + + result = await self.execute_query(query, use_query_options=False) + return list(result.scalars().all()) + async def get_distinct_directories(self) -> List[str]: """Extract unique directory paths from file_path column. diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index b22c02a4..4a00d510 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -1,12 +1,17 @@ """Service for file operations with checksum tracking.""" +import asyncio +import hashlib import mimetypes from os import stat_result from pathlib import Path from typing import Any, Dict, Tuple, Union +import aiofiles +import yaml + from basic_memory import file_utils -from basic_memory.file_utils import FileError +from basic_memory.file_utils import FileError, ParseError from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.models import Entity as EntityModel from basic_memory.schemas import Entity as EntitySchema @@ -16,13 +21,15 @@ from loguru import logger class FileService: - """Service for handling file operations. + """Service for handling file operations with concurrency control. All paths are handled as Path objects internally. Strings are converted to Path objects when passed in. Relative paths are assumed to be relative to base_path. Features: + - True async I/O with aiofiles (non-blocking) + - Built-in concurrency limits (semaphore) - Consistent file writing with checksums - Frontmatter management - Atomic operations @@ -33,9 +40,13 @@ class FileService: self, base_path: Path, markdown_processor: MarkdownProcessor, + max_concurrent_files: int = 10, ): self.base_path = base_path.resolve() # Get absolute path self.markdown_processor = markdown_processor + # Semaphore to limit concurrent file operations + # Prevents OOM on large projects by processing files in batches + self._file_semaphore = asyncio.Semaphore(max_concurrent_files) def get_entity_path(self, entity: Union[EntityModel, EntitySchema]) -> Path: """Generate absolute filesystem path for entity. @@ -104,6 +115,33 @@ class FileService: logger.error("Failed to check file existence", path=str(path), error=str(e)) raise FileOperationError(f"Failed to check file existence: {e}") + async def ensure_directory(self, path: FilePath) -> None: + """Ensure directory exists, creating if necessary. + + Uses semaphore to control concurrency for directory creation operations. + + Args: + path: Directory path to ensure (Path or string) + + Raises: + FileOperationError: If directory creation fails + """ + try: + # Convert string to Path if needed + path_obj = self.base_path / path if isinstance(path, str) else path + full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj + + # Use semaphore for concurrency control + async with self._file_semaphore: + # Run blocking mkdir in thread pool + loop = asyncio.get_event_loop() + await loop.run_in_executor( + None, lambda: full_path.mkdir(parents=True, exist_ok=True) + ) + except Exception as e: # pragma: no cover + logger.error("Failed to create directory", path=str(path), error=str(e)) + raise FileOperationError(f"Failed to create directory {path}: {e}") + async def write_file(self, path: FilePath, content: str) -> str: """Write content to file and return checksum. @@ -126,7 +164,7 @@ class FileService: try: # Ensure parent directory exists - await file_utils.ensure_directory(full_path.parent) + await self.ensure_directory(full_path.parent) # Write content atomically logger.info( @@ -147,9 +185,45 @@ class FileService: logger.exception("File write error", path=str(full_path), error=str(e)) raise FileOperationError(f"Failed to write file: {e}") - # TODO remove read_file + async def read_file_content(self, path: FilePath) -> str: + """Read file content using true async I/O with aiofiles. + + Handles both absolute and relative paths. Relative paths are resolved + against base_path. + + Args: + path: Path to read (Path or string) + + Returns: + File content as string + + Raises: + FileOperationError: If read fails + """ + # Convert string to Path if needed + path_obj = self.base_path / path if isinstance(path, str) else path + full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj + + try: + logger.debug("Reading file content", operation="read_file_content", path=str(full_path)) + async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f: + content = await f.read() + + logger.debug( + "File read completed", + path=str(full_path), + content_length=len(content), + ) + return content + + except Exception as e: + logger.exception("File read error", path=str(full_path), error=str(e)) + raise FileOperationError(f"Failed to read file: {e}") + async def read_file(self, path: FilePath) -> Tuple[str, str]: - """Read file and compute checksum. + """Read file and compute checksum using true async I/O. + + Uses aiofiles for non-blocking file reads. Handles both absolute and relative paths. Relative paths are resolved against base_path. @@ -169,7 +243,11 @@ class FileService: try: logger.debug("Reading file", operation="read_file", path=str(full_path)) - content = full_path.read_text(encoding="utf-8") + + # Use aiofiles for non-blocking read + async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f: + content = await f.read() + checksum = await file_utils.compute_checksum(content) logger.debug( @@ -199,29 +277,85 @@ class FileService: full_path.unlink(missing_ok=True) async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str: - """ - Update frontmatter fields in a file while preserving all content. + """Update frontmatter fields in a file while preserving all content. + + Only modifies the frontmatter section, leaving all content untouched. + Creates frontmatter section if none exists. + Returns checksum of updated file. + + Uses aiofiles for true async I/O (non-blocking). Args: - path: Path to the file (Path or string) - updates: Dictionary of frontmatter fields to update + path: Path to markdown file (Path or string) + updates: Dict of frontmatter fields to update Returns: Checksum of updated file + + Raises: + FileOperationError: If file operations fail + ParseError: If frontmatter parsing fails """ # Convert string to Path if needed path_obj = self.base_path / path if isinstance(path, str) else path full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj - return await file_utils.update_frontmatter(full_path, updates) + + try: + # Read current content using aiofiles + async with aiofiles.open(full_path, mode="r", encoding="utf-8") as f: + content = await f.read() + + # Parse current frontmatter with proper error handling for malformed YAML + current_fm = {} + if file_utils.has_frontmatter(content): + try: + current_fm = file_utils.parse_frontmatter(content) + content = file_utils.remove_frontmatter(content) + except (ParseError, yaml.YAMLError) as e: + # Log warning and treat as plain markdown without frontmatter + logger.warning( + f"Failed to parse YAML frontmatter in {full_path}: {e}. " + "Treating file as plain markdown without frontmatter." + ) + # Keep full content, treat as having no frontmatter + current_fm = {} + + # Update frontmatter + new_fm = {**current_fm, **updates} + + # Write new file with updated frontmatter + yaml_fm = yaml.dump(new_fm, sort_keys=False, allow_unicode=True) + final_content = f"---\n{yaml_fm}---\n\n{content.strip()}" + + logger.debug( + "Updating frontmatter", path=str(full_path), update_keys=list(updates.keys()) + ) + + await file_utils.write_file_atomic(full_path, final_content) + return await file_utils.compute_checksum(final_content) + + except Exception as e: + # Only log real errors (not YAML parsing, which is handled above) + if not isinstance(e, (ParseError, yaml.YAMLError)): + logger.error( + "Failed to update frontmatter", + path=str(full_path), + error=str(e), + ) + raise FileOperationError(f"Failed to update frontmatter: {e}") async def compute_checksum(self, path: FilePath) -> str: - """Compute checksum for a file. + """Compute checksum for a file using true async I/O. + + Uses aiofiles for non-blocking I/O with 64KB chunked reading. + Semaphore limits concurrent file operations to prevent OOM. + Memory usage is constant regardless of file size. Args: path: Path to the file (Path or string) Returns: - Checksum of the file content + SHA256 checksum hex string Raises: FileError: If checksum computation fails @@ -230,18 +364,22 @@ class FileService: path_obj = self.base_path / path if isinstance(path, str) else path full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj - try: - if self.is_markdown(path): - # read str - content = full_path.read_text(encoding="utf-8") - else: - # read bytes - content = full_path.read_bytes() - return await file_utils.compute_checksum(content) + # Semaphore controls concurrency - max N files processed at once + async with self._file_semaphore: + try: + hasher = hashlib.sha256() + chunk_size = 65536 # 64KB chunks - except Exception as e: # pragma: no cover - logger.error("Failed to compute checksum", path=str(full_path), error=str(e)) - raise FileError(f"Failed to compute checksum for {path}: {e}") + # async I/O with aiofiles + async with aiofiles.open(full_path, mode="rb") as f: + while chunk := await f.read(chunk_size): + hasher.update(chunk) + + return hasher.hexdigest() + + except Exception as e: # pragma: no cover + logger.error("Failed to compute checksum", path=str(full_path), error=str(e)) + raise FileError(f"Failed to compute checksum for {path}: {e}") def file_stats(self, path: FilePath) -> stat_result: """Return file stats for a given path. diff --git a/src/basic_memory/services/initialization.py b/src/basic_memory/services/initialization.py index f45aa632..9ed651ed 100644 --- a/src/basic_memory/services/initialization.py +++ b/src/basic_memory/services/initialization.py @@ -118,18 +118,8 @@ async def initialize_file_sync( sync_dir = Path(project.path) await sync_service.sync(sync_dir, project_name=project.name) logger.info(f"Background sync completed successfully for project: {project.name}") - - # Mark project as watching for changes after successful sync - from basic_memory.services.sync_status_service import sync_status_tracker - - sync_status_tracker.start_project_watch(project.name) - logger.info(f"Project {project.name} is now watching for changes") except Exception as e: # pragma: no cover logger.error(f"Error in background sync for project {project.name}: {e}") - # Mark sync as failed for this project - from basic_memory.services.sync_status_service import sync_status_tracker - - sync_status_tracker.fail_project_sync(project.name, str(e)) # Create background tasks for all project syncs (non-blocking) sync_tasks = [ diff --git a/src/basic_memory/services/sync_status_service.py b/src/basic_memory/services/sync_status_service.py deleted file mode 100644 index 0781b6d9..00000000 --- a/src/basic_memory/services/sync_status_service.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Simple sync status tracking service.""" - -from dataclasses import dataclass -from enum import Enum -from typing import Dict, Optional - - -class SyncStatus(Enum): - """Status of sync operations.""" - - IDLE = "idle" - SCANNING = "scanning" - SYNCING = "syncing" - COMPLETED = "completed" - FAILED = "failed" - WATCHING = "watching" - - -@dataclass -class ProjectSyncStatus: - """Sync status for a single project.""" - - project_name: str - status: SyncStatus - message: str = "" - files_total: int = 0 - files_processed: int = 0 - error: Optional[str] = None - - -class SyncStatusTracker: - """Global tracker for all sync operations.""" - - def __init__(self): - self._project_statuses: Dict[str, ProjectSyncStatus] = {} - self._global_status: SyncStatus = SyncStatus.IDLE - - def start_project_sync(self, project_name: str, files_total: int = 0) -> None: - """Start tracking sync for a project.""" - self._project_statuses[project_name] = ProjectSyncStatus( - project_name=project_name, - status=SyncStatus.SCANNING, - message="Scanning files", - files_total=files_total, - files_processed=0, - ) - self._update_global_status() - - def update_project_progress( # pragma: no cover - self, - project_name: str, - status: SyncStatus, - message: str = "", - files_processed: int = 0, - files_total: Optional[int] = None, - ) -> None: - """Update progress for a project.""" - if project_name not in self._project_statuses: # pragma: no cover - return - - project_status = self._project_statuses[project_name] - project_status.status = status - project_status.message = message - project_status.files_processed = files_processed - - if files_total is not None: - project_status.files_total = files_total - - self._update_global_status() - - def complete_project_sync(self, project_name: str) -> None: - """Mark project sync as completed.""" - if project_name in self._project_statuses: - self._project_statuses[project_name].status = SyncStatus.COMPLETED - self._project_statuses[project_name].message = "Sync completed" - self._update_global_status() - - def fail_project_sync(self, project_name: str, error: str) -> None: - """Mark project sync as failed.""" - if project_name in self._project_statuses: - self._project_statuses[project_name].status = SyncStatus.FAILED - self._project_statuses[project_name].error = error - self._update_global_status() - - def start_project_watch(self, project_name: str) -> None: - """Mark project as watching for changes (steady state after sync).""" - if project_name in self._project_statuses: - self._project_statuses[project_name].status = SyncStatus.WATCHING - self._project_statuses[project_name].message = "Watching for changes" - self._update_global_status() - else: - # Create new status if project isn't tracked yet - self._project_statuses[project_name] = ProjectSyncStatus( - project_name=project_name, - status=SyncStatus.WATCHING, - message="Watching for changes", - files_total=0, - files_processed=0, - ) - self._update_global_status() - - def _update_global_status(self) -> None: - """Update global status based on project statuses.""" - if not self._project_statuses: # pragma: no cover - self._global_status = SyncStatus.IDLE - return - - statuses = [p.status for p in self._project_statuses.values()] - - if any(s == SyncStatus.FAILED for s in statuses): - self._global_status = SyncStatus.FAILED - elif any(s in (SyncStatus.SCANNING, SyncStatus.SYNCING) for s in statuses): - self._global_status = SyncStatus.SYNCING - elif all(s in (SyncStatus.COMPLETED, SyncStatus.WATCHING) for s in statuses): - self._global_status = SyncStatus.COMPLETED - else: - self._global_status = SyncStatus.SYNCING - - @property - def global_status(self) -> SyncStatus: - """Get overall sync status.""" - return self._global_status - - @property - def is_syncing(self) -> bool: - """Check if any sync operation is in progress.""" - return self._global_status in (SyncStatus.SCANNING, SyncStatus.SYNCING) - - @property - def is_ready(self) -> bool: # pragma: no cover - """Check if system is ready (no sync in progress).""" - return self._global_status in (SyncStatus.IDLE, SyncStatus.COMPLETED) - - def is_project_ready(self, project_name: str) -> bool: - """Check if a specific project is ready for operations. - - Args: - project_name: Name of the project to check - - Returns: - True if the project is ready (completed, watching, or not tracked), - False if the project is syncing, scanning, or failed - """ - project_status = self._project_statuses.get(project_name) - if not project_status: - # Project not tracked = ready (likely hasn't been synced yet) - return True - - return project_status.status in (SyncStatus.COMPLETED, SyncStatus.WATCHING, SyncStatus.IDLE) - - def get_project_status(self, project_name: str) -> Optional[ProjectSyncStatus]: - """Get status for a specific project.""" - return self._project_statuses.get(project_name) - - def get_all_projects(self) -> Dict[str, ProjectSyncStatus]: - """Get all project statuses.""" - return self._project_statuses.copy() - - def get_summary(self) -> str: # pragma: no cover - """Get a user-friendly summary of sync status.""" - if self._global_status == SyncStatus.IDLE: - return "✅ System ready" - elif self._global_status == SyncStatus.COMPLETED: - return "✅ All projects synced successfully" - elif self._global_status == SyncStatus.FAILED: - failed_projects = [ - p.project_name - for p in self._project_statuses.values() - if p.status == SyncStatus.FAILED - ] - return f"❌ Sync failed for: {', '.join(failed_projects)}" - else: - active_projects = [ - p.project_name - for p in self._project_statuses.values() - if p.status in (SyncStatus.SCANNING, SyncStatus.SYNCING) - ] - total_files = sum(p.files_total for p in self._project_statuses.values()) - processed_files = sum(p.files_processed for p in self._project_statuses.values()) - - if total_files > 0: - progress_pct = (processed_files / total_files) * 100 - return f"🔄 Syncing {len(active_projects)} projects ({processed_files}/{total_files} files, {progress_pct:.0f}%)" - else: - return f"🔄 Syncing {len(active_projects)} projects" - - def clear_completed(self) -> None: - """Remove completed project statuses to clean up memory.""" - self._project_statuses = { - name: status - for name, status in self._project_statuses.items() - if status.status != SyncStatus.COMPLETED - } - self._update_global_status() - - -# Global sync status tracker instance -sync_status_tracker = SyncStatusTracker() diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index 506fd437..93941d79 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -4,14 +4,14 @@ import asyncio import os import time from collections import OrderedDict -from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import AsyncIterator, Dict, List, Optional, Set, Tuple +import aiofiles.os +import logfire from loguru import logger -from sqlalchemy import select from sqlalchemy.exc import IntegrityError from basic_memory import db @@ -20,13 +20,17 @@ from basic_memory.file_utils import has_frontmatter from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path from basic_memory.markdown import EntityParser, MarkdownProcessor from basic_memory.models import Entity, Project -from basic_memory.repository import EntityRepository, RelationRepository, ObservationRepository +from basic_memory.repository import ( + EntityRepository, + RelationRepository, + ObservationRepository, + ProjectRepository, +) from basic_memory.repository.search_repository import SearchRepository from basic_memory.services import EntityService, FileService from basic_memory.services.exceptions import SyncFatalError from basic_memory.services.link_resolver import LinkResolver from basic_memory.services.search_service import SearchService -from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus # Circuit breaker configuration MAX_CONSECUTIVE_FAILURES = 3 @@ -120,6 +124,7 @@ class SyncService: entity_parser: EntityParser, entity_repository: EntityRepository, relation_repository: RelationRepository, + project_repository: ProjectRepository, search_service: SearchService, file_service: FileService, ): @@ -128,60 +133,15 @@ class SyncService: self.entity_parser = entity_parser self.entity_repository = entity_repository self.relation_repository = relation_repository + self.project_repository = project_repository self.search_service = search_service self.file_service = file_service - self._thread_pool = ThreadPoolExecutor(max_workers=app_config.sync_thread_pool_size) # Load ignore patterns once at initialization for performance self._ignore_patterns = load_bmignore_patterns() # Circuit breaker: track file failures to prevent infinite retry loops # Use OrderedDict for LRU behavior with bounded size to prevent unbounded memory growth self._file_failures: OrderedDict[str, FileFailureInfo] = OrderedDict() self._max_tracked_failures = 100 # Limit failure cache size - # Semaphore to limit concurrent file operations and prevent OOM on large projects - # Limits peak memory usage by processing files in batches rather than all at once - self._file_semaphore = asyncio.Semaphore(app_config.sync_max_concurrent_files) - - async def _read_file_async(self, file_path: Path) -> str: - """Read file content in thread pool to avoid blocking the event loop. - - Uses semaphore to limit concurrent file reads and prevent OOM on large projects. - """ - async with self._file_semaphore: - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._thread_pool, file_path.read_text, "utf-8") - - async def _compute_checksum_async(self, path: str) -> str: - """Compute file checksum in thread pool to avoid blocking the event loop. - - Uses semaphore to limit concurrent file reads and prevent OOM on large projects. - """ - - def _sync_compute_checksum(path_str: str) -> str: - # Synchronous version for thread pool execution - path_obj = self.file_service.base_path / path_str - - if self.file_service.is_markdown(path_str): - content = path_obj.read_text(encoding="utf-8") - else: - content = path_obj.read_bytes() - - # Use the synchronous version of compute_checksum - import hashlib - - if isinstance(content, str): - content_bytes = content.encode("utf-8") - else: - content_bytes = content - return hashlib.sha256(content_bytes).hexdigest() - - async with self._file_semaphore: - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._thread_pool, _sync_compute_checksum, path) - - def __del__(self): - """Cleanup thread pool when service is destroyed.""" - if hasattr(self, "_thread_pool"): - self._thread_pool.shutdown(wait=False) async def _should_skip_file(self, path: str) -> bool: """Check if file should be skipped due to repeated failures. @@ -206,7 +166,7 @@ class SyncService: # Compute current checksum to see if file changed try: - current_checksum = await self._compute_checksum_async(path) + current_checksum = await self.file_service.compute_checksum(path) # If checksum changed, file was modified - reset and retry if current_checksum != failure_info.last_checksum: @@ -236,7 +196,7 @@ class SyncService: # Compute checksum for failure tracking try: - checksum = await self._compute_checksum_async(path) + checksum = await self.file_service.compute_checksum(path) except Exception: # If checksum fails, use empty string (better than crashing) checksum = "" @@ -255,12 +215,17 @@ class SyncService: f"path={path}, error={error}" ) + # Record metric for file failure + logfire.metric_counter("sync.circuit_breaker.failures").add(1) + # Log when threshold is reached if failure_info.count >= MAX_CONSECUTIVE_FAILURES: logger.error( f"File {path} has failed {MAX_CONSECUTIVE_FAILURES} times and will be skipped. " f"First failure: {failure_info.first_failure}, Last error: {error}" ) + # Record metric for file being blocked by circuit breaker + logfire.metric_counter("sync.circuit_breaker.blocked_files").add(1) else: # Create new failure record self._file_failures[path] = FileFailureInfo( @@ -290,128 +255,113 @@ class SyncService: logger.info(f"Clearing failure history for {path} after successful sync") del self._file_failures[path] + @logfire.instrument() async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport: - """Sync all files with database.""" + """Sync all files with database and update scan watermark.""" start_time = time.time() + sync_start_timestamp = time.time() # Capture at start for watermark logger.info(f"Sync operation started for directory: {directory}") - # Start tracking sync for this project if project name provided - if project_name: - sync_status_tracker.start_project_sync(project_name) - # initial paths from db to sync # path -> checksum report = await self.scan(directory) - # Update progress with file counts - if project_name: - sync_status_tracker.update_project_progress( - project_name=project_name, - status=SyncStatus.SYNCING, - message="Processing file changes", - files_total=report.total, - files_processed=0, - ) - # order of sync matters to resolve relations effectively logger.info( f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, " + f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}" ) - files_processed = 0 - # sync moves first - for old_path, new_path in report.moves.items(): - # in the case where a file has been deleted and replaced by another file - # it will show up in the move and modified lists, so handle it in modified - if new_path in report.modified: - report.modified.remove(new_path) - logger.debug( - f"File marked as moved and modified: old_path={old_path}, new_path={new_path}" - ) - else: - await self.handle_move(old_path, new_path) - - files_processed += 1 - if project_name: - sync_status_tracker.update_project_progress( # pragma: no cover - project_name=project_name, - status=SyncStatus.SYNCING, - message="Processing moves", - files_processed=files_processed, - ) + with logfire.span("process_moves", move_count=len(report.moves)): + for old_path, new_path in report.moves.items(): + # in the case where a file has been deleted and replaced by another file + # it will show up in the move and modified lists, so handle it in modified + if new_path in report.modified: + report.modified.remove(new_path) + logger.debug( + f"File marked as moved and modified: old_path={old_path}, new_path={new_path}" + ) + else: + await self.handle_move(old_path, new_path) # deleted next - for path in report.deleted: - await self.handle_delete(path) - files_processed += 1 - if project_name: - sync_status_tracker.update_project_progress( # pragma: no cover - project_name=project_name, - status=SyncStatus.SYNCING, - message="Processing deletions", - files_processed=files_processed, - ) + with logfire.span("process_deletes", delete_count=len(report.deleted)): + for path in report.deleted: + await self.handle_delete(path) # then new and modified - for path in report.new: - entity, _ = await self.sync_file(path, new=True) + with logfire.span("process_new_files", new_count=len(report.new)): + for path in report.new: + entity, _ = await self.sync_file(path, new=True) - # Track if file was skipped - if entity is None and await self._should_skip_file(path): - failure_info = self._file_failures[path] - report.skipped_files.append( - SkippedFile( - path=path, - reason=failure_info.last_error, - failure_count=failure_info.count, - first_failed=failure_info.first_failure, + # Track if file was skipped + if entity is None and await self._should_skip_file(path): + failure_info = self._file_failures[path] + report.skipped_files.append( + SkippedFile( + path=path, + reason=failure_info.last_error, + failure_count=failure_info.count, + first_failed=failure_info.first_failure, + ) ) - ) - files_processed += 1 - if project_name: - sync_status_tracker.update_project_progress( - project_name=project_name, - status=SyncStatus.SYNCING, - message="Processing new files", - files_processed=files_processed, - ) + with logfire.span("process_modified_files", modified_count=len(report.modified)): + for path in report.modified: + entity, _ = await self.sync_file(path, new=False) - for path in report.modified: - entity, _ = await self.sync_file(path, new=False) - - # Track if file was skipped - if entity is None and await self._should_skip_file(path): - failure_info = self._file_failures[path] - report.skipped_files.append( - SkippedFile( - path=path, - reason=failure_info.last_error, - failure_count=failure_info.count, - first_failed=failure_info.first_failure, + # Track if file was skipped + if entity is None and await self._should_skip_file(path): + failure_info = self._file_failures[path] + report.skipped_files.append( + SkippedFile( + path=path, + reason=failure_info.last_error, + failure_count=failure_info.count, + first_failed=failure_info.first_failure, + ) ) + + # Only resolve relations if there were actual changes + # If no files changed, no new unresolved relations could have been created + with logfire.span("resolve_relations"): + if report.total > 0: + await self.resolve_relations() + else: + logger.info("Skipping relation resolution - no file changes detected") + + # Update scan watermark after successful sync + # Use the timestamp from sync start (not end) to ensure we catch files + # created during the sync on the next iteration + current_file_count = await self._quick_count_files(directory) + if self.entity_repository.project_id is not None: + project = await self.project_repository.find_by_id(self.entity_repository.project_id) + if project: + await self.project_repository.update( + project.id, + { + "last_scan_timestamp": sync_start_timestamp, + "last_file_count": current_file_count, + }, ) - - files_processed += 1 - if project_name: - sync_status_tracker.update_project_progress( # pragma: no cover - project_name=project_name, - status=SyncStatus.SYNCING, - message="Processing modified files", - files_processed=files_processed, + logger.debug( + f"Updated scan watermark: timestamp={sync_start_timestamp}, " + f"file_count={current_file_count}" ) - await self.resolve_relations() - - # Mark sync as completed - if project_name: - sync_status_tracker.complete_project_sync(project_name) - duration_ms = int((time.time() - start_time) * 1000) + # Record metrics for sync operation + logfire.metric_histogram("sync.duration", unit="ms").record(duration_ms) + logfire.metric_counter("sync.files.new").add(len(report.new)) + logfire.metric_counter("sync.files.modified").add(len(report.modified)) + logfire.metric_counter("sync.files.deleted").add(len(report.deleted)) + logfire.metric_counter("sync.files.moved").add(len(report.moves)) + if report.skipped_files: + logfire.metric_counter("sync.files.skipped").add(len(report.skipped_files)) + # Log summary with skipped files if any if report.skipped_files: logger.warning( @@ -432,75 +382,206 @@ class SyncService: return report + @logfire.instrument() async def scan(self, directory): - """Scan directory for changes compared to database state.""" + """Smart scan using watermark and file count for large project optimization. - db_paths = await self.get_db_file_state() - logger.info(f"Scanning directory {directory}. Found {len(db_paths)} db paths") + Uses scan watermark tracking to dramatically reduce scan time for large projects: + - Tracks last_scan_timestamp and last_file_count in Project model + - Uses `find -newermt` for incremental scanning (only changed files) + - Falls back to full scan when deletions detected (file count decreased) + + Expected performance: + - No changes: 225x faster (2s vs 450s for 1,460 files on TigrisFS) + - Few changes: 84x faster (5s vs 420s) + - Deletions: Full scan (rare, acceptable) + + Architecture: + - Get current file count quickly (find | wc -l: 1.4s) + - Compare with last_file_count to detect deletions + - If no deletions: incremental scan with find -newermt (0.2s) + - Process changed files with mtime-based comparison + """ + scan_start_time = time.time() - # Track potentially moved files by checksum - scan_result = await self.scan_directory(directory) report = SyncReport() - # First find potential new files and record checksums - # if a path is not present in the db, it could be new or could be the destination of a move - for file_path, checksum in scan_result.files.items(): - if file_path not in db_paths: - report.new.add(file_path) - report.checksums[file_path] = checksum + # Get current project to check watermark + if self.entity_repository.project_id is None: + raise ValueError("Entity repository has no project_id set") - # Now detect moves and deletions - for db_path, db_checksum in db_paths.items(): - local_checksum_for_db_path = scan_result.files.get(db_path) + project = await self.project_repository.find_by_id(self.entity_repository.project_id) + if project is None: + raise ValueError(f"Project not found: {self.entity_repository.project_id}") - # file not modified - if db_checksum == local_checksum_for_db_path: - pass + # Step 1: Quick file count + logger.debug("Counting files in directory") + current_count = await self._quick_count_files(directory) + logger.debug(f"Found {current_count} files in directory") - # if checksums don't match for the same path, its modified - if local_checksum_for_db_path and db_checksum != local_checksum_for_db_path: - report.modified.add(db_path) - report.checksums[db_path] = local_checksum_for_db_path + # Step 2: Determine scan strategy based on watermark and file count + if project.last_file_count is None: + # First sync ever → full scan + scan_type = "full_initial" + logger.info("First sync for this project, performing full scan") + file_paths_to_scan = await self._scan_directory_full(directory) - # check if it's moved or deleted - if not local_checksum_for_db_path: - # if we find the checksum in another file, it's a move - if db_checksum in scan_result.checksums: - new_path = scan_result.checksums[db_checksum] - report.moves[db_path] = new_path + elif current_count < project.last_file_count: + # Files deleted → need full scan to detect which ones + scan_type = "full_deletions" + logger.info( + f"File count decreased ({project.last_file_count} → {current_count}), " + f"running full scan to detect deletions" + ) + file_paths_to_scan = await self._scan_directory_full(directory) - # Remove from new files if present - if new_path in report.new: - report.new.remove(new_path) + elif project.last_scan_timestamp is not None: + # Incremental scan: only files modified since last scan + scan_type = "incremental" + logger.info( + f"Running incremental scan for files modified since {project.last_scan_timestamp}" + ) + file_paths_to_scan = await self._scan_directory_modified_since( + directory, project.last_scan_timestamp + ) + logger.info( + f"Incremental scan found {len(file_paths_to_scan)} potentially changed files" + ) - # deleted - else: - report.deleted.add(db_path) - logger.info(f"Completed scan for directory {directory}, found {report.total} changes.") - return report + else: + # Fallback to full scan (no watermark available) + scan_type = "full_fallback" + logger.warning("No scan watermark available, falling back to full scan") + file_paths_to_scan = await self._scan_directory_full(directory) - async def get_db_file_state(self) -> Dict[str, str]: - """Get file_path and checksums from database. - - Optimized to query only the columns we need (file_path, checksum) without - loading full entities or their relationships. This is 10-100x faster for - large projects compared to loading all entities with observations/relations. - - Returns: - Dict mapping file paths to checksums - """ - # Query only the columns we need - no entity objects or relationships - query = select(Entity.file_path, Entity.checksum).where( - Entity.project_id == self.entity_repository.project_id + # Record scan type metric + logfire.metric_counter(f"sync.scan.{scan_type}").add(1) + logfire.metric_histogram("sync.scan.files_scanned", unit="files").record( + len(file_paths_to_scan) ) - async with db.scoped_session(self.entity_repository.session_maker) as session: - result = await session.execute(query) - rows = result.all() + # Step 3: Process each file with mtime-based comparison + scanned_paths: Set[str] = set() + changed_checksums: Dict[str, str] = {} - logger.info(f"Found {len(rows)} db file records") - return {row.file_path: row.checksum or "" for row in rows} + logger.debug(f"Processing {len(file_paths_to_scan)} files with mtime-based comparison") + for rel_path in file_paths_to_scan: + scanned_paths.add(rel_path) + + # Get file stats + abs_path = directory / rel_path + if not abs_path.exists(): + # File was deleted between scan and now (race condition) + continue + + stat_info = abs_path.stat() + + # Indexed lookup - single file query (not full table scan) + db_entity = await self.entity_repository.get_by_file_path(rel_path) + + if db_entity is None: + # New file - need checksum for move detection + checksum = await self.file_service.compute_checksum(rel_path) + report.new.add(rel_path) + changed_checksums[rel_path] = checksum + logger.trace(f"New file detected: {rel_path}") + continue + + # File exists in DB - check if mtime/size changed + db_mtime = db_entity.mtime + db_size = db_entity.size + fs_mtime = stat_info.st_mtime + fs_size = stat_info.st_size + + # Compare mtime and size (like rsync/rclone) + # Allow small epsilon for float comparison (0.01s = 10ms) + mtime_changed = db_mtime is None or abs(fs_mtime - db_mtime) > 0.01 + size_changed = db_size is None or fs_size != db_size + + if mtime_changed or size_changed: + # File modified - compute checksum + checksum = await self.file_service.compute_checksum(rel_path) + db_checksum = db_entity.checksum + + # Only mark as modified if checksum actually differs + # (handles cases where mtime changed but content didn't, e.g., git operations) + if checksum != db_checksum: + report.modified.add(rel_path) + changed_checksums[rel_path] = checksum + logger.trace( + f"Modified file detected: {rel_path}, " + f"mtime_changed={mtime_changed}, size_changed={size_changed}" + ) + else: + # File unchanged - no checksum needed + logger.trace(f"File unchanged (mtime/size match): {rel_path}") + + # Step 4: Detect moves (for both full and incremental scans) + # Check if any "new" files are actually moves by matching checksums + for new_path in list(report.new): # Use list() to allow modification during iteration + new_checksum = changed_checksums.get(new_path) + if not new_checksum: + continue + + # Look for existing entity with same checksum but different path + # This could be a move or a copy + existing_entities = await self.entity_repository.find_by_checksum(new_checksum) + + for candidate in existing_entities: + if candidate.file_path == new_path: + # Same path, skip (shouldn't happen for "new" files but be safe) + continue + + # Check if the old path still exists on disk + old_path_abs = directory / candidate.file_path + if old_path_abs.exists(): + # Original still exists → this is a copy, not a move + logger.trace( + f"File copy detected (not move): {candidate.file_path} copied to {new_path}" + ) + continue + + # Original doesn't exist → this is a move! + report.moves[candidate.file_path] = new_path + report.new.remove(new_path) + logger.trace(f"Move detected: {candidate.file_path} -> {new_path}") + break # Only match first candidate + + # Step 5: Detect deletions (only for full scans) + # Incremental scans can't reliably detect deletions since they only see modified files + if scan_type in ("full_initial", "full_deletions", "full_fallback"): + # Use optimized query for just file paths (not full entities) + db_file_paths = await self.entity_repository.get_all_file_paths() + logger.debug(f"Found {len(db_file_paths)} db paths for deletion detection") + + for db_path in db_file_paths: + if db_path not in scanned_paths: + # File in DB but not on filesystem + # Check if it was already detected as a move + if db_path in report.moves: + # Already handled as a move, skip + continue + + # File was deleted + report.deleted.add(db_path) + logger.trace(f"Deleted file detected: {db_path}") + + # Store checksums for files that need syncing + report.checksums = changed_checksums + + scan_duration_ms = int((time.time() - scan_start_time) * 1000) + logfire.metric_histogram("sync.scan.duration", unit="ms").record(scan_duration_ms) + + logger.info( + f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, " + f"found {report.total} changes (new={len(report.new)}, " + f"modified={len(report.modified)}, deleted={len(report.deleted)}, " + f"moves={len(report.moves)})" + ) + return report + + @logfire.instrument() async def sync_file( self, path: str, new: bool = True ) -> Tuple[Optional[Entity], Optional[str]]: @@ -555,6 +636,7 @@ class SyncService: return None, None + @logfire.instrument() async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]: """Sync a markdown file with full processing. @@ -568,8 +650,7 @@ class SyncService: # Parse markdown first to get any existing permalink logger.debug(f"Parsing markdown file, path: {path}, new: {new}") - file_path = self.entity_parser.base_path / path - file_content = await self._read_file_async(file_path) + file_content = await self.file_service.read_file_content(path) file_contains_frontmatter = has_frontmatter(file_content) # Get file timestamps for tracking modification times @@ -613,12 +694,20 @@ class SyncService: # After updating relations, we need to compute the checksum again # This is necessary for files with wikilinks to ensure consistent checksums # after relation processing is complete - final_checksum = await self._compute_checksum_async(path) + final_checksum = await self.file_service.compute_checksum(path) - # Update checksum and timestamps from file system + # Update checksum, timestamps, and file metadata from file system + # Store mtime/size for efficient change detection in future scans # This ensures temporal ordering in search and recent activity uses actual file modification times await self.entity_repository.update( - entity.id, {"checksum": final_checksum, "created_at": created, "updated_at": modified} + entity.id, + { + "checksum": final_checksum, + "created_at": created, + "updated_at": modified, + "mtime": file_stats.st_mtime, + "size": file_stats.st_size, + }, ) logger.debug( @@ -630,6 +719,7 @@ class SyncService: # Return the final checksum to ensure everything is consistent return entity, final_checksum + @logfire.instrument() async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]: """Sync a non-markdown file with basic tracking. @@ -640,7 +730,7 @@ class SyncService: Returns: Tuple of (entity, checksum) """ - checksum = await self._compute_checksum_async(path) + checksum = await self.file_service.compute_checksum(path) if new: # Generate permalink from path - skip conflict checks during bulk sync await self.entity_service.resolve_permalink(path, skip_conflict_check=True) @@ -664,6 +754,8 @@ class SyncService: created_at=created, updated_at=modified, content_type=content_type, + mtime=file_stats.st_mtime, + size=file_stats.st_size, ) ) return entity, checksum @@ -679,8 +771,16 @@ class SyncService: logger.error(f"Entity not found after constraint violation, path={path}") raise ValueError(f"Entity not found after constraint violation: {path}") + # Re-get file stats since we're in update path + file_stats_for_update = self.file_service.file_stats(path) updated = await self.entity_repository.update( - entity.id, {"file_path": path, "checksum": checksum} + entity.id, + { + "file_path": path, + "checksum": checksum, + "mtime": file_stats_for_update.st_mtime, + "size": file_stats_for_update.st_size, + }, ) if updated is None: # pragma: no cover @@ -701,9 +801,17 @@ class SyncService: logger.error(f"Entity not found for existing file, path={path}") raise ValueError(f"Entity not found for existing file: {path}") - # Update checksum and modification time from file system + # Update checksum, modification time, and file metadata from file system + # Store mtime/size for efficient change detection in future scans updated = await self.entity_repository.update( - entity.id, {"file_path": path, "checksum": checksum, "updated_at": modified} + entity.id, + { + "file_path": path, + "checksum": checksum, + "updated_at": modified, + "mtime": file_stats.st_mtime, + "size": file_stats.st_size, + }, ) if updated is None: # pragma: no cover @@ -712,6 +820,7 @@ class SyncService: return updated, checksum + @logfire.instrument() async def handle_delete(self, file_path: str): """Handle complete entity deletion including search index cleanup.""" @@ -743,6 +852,7 @@ class SyncService: else: await self.search_service.delete_by_entity_id(entity.id) + @logfire.instrument() async def handle_move(self, old_path, new_path): logger.debug("Moving entity", old_path=old_path, new_path=new_path) @@ -847,6 +957,7 @@ class SyncService: # update search index await self.search_service.index_entity(updated) + @logfire.instrument() async def resolve_relations(self, entity_id: int | None = None): """Try to resolve unresolved relations. @@ -908,64 +1019,163 @@ class SyncService: # update search index await self.search_service.index_entity(resolved_entity) - async def scan_directory(self, directory: Path) -> ScanResult: + async def _quick_count_files(self, directory: Path) -> int: + """Fast file count using find command. + + Uses subprocess to leverage OS-level file counting which is much faster + than Python iteration, especially on network filesystems like TigrisFS. + + Args: + directory: Directory to count files in + + Returns: + Number of files in directory (recursive) """ - Scan directory for markdown files and their checksums. + process = await asyncio.create_subprocess_shell( + f'find "{directory}" -type f | wc -l', + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + + if process.returncode != 0: + error_msg = stderr.decode().strip() + logger.error( + f"FILE COUNT OPTIMIZATION FAILED: find command failed with exit code {process.returncode}, " + f"error: {error_msg}. Falling back to manual count. " + f"This will slow down watermark detection!" + ) + # Track optimization failures for visibility + logfire.metric_counter("sync.scan.file_count_failure").add(1) + # Fallback: count using scan_directory + count = 0 + async for _ in self.scan_directory(directory): + count += 1 + return count + + return int(stdout.strip()) + + async def _scan_directory_modified_since( + self, directory: Path, since_timestamp: float + ) -> List[str]: + """Use find -newermt for filesystem-level filtering of modified files. + + This is dramatically faster than scanning all files and comparing mtimes, + especially on network filesystems like TigrisFS where stat operations are expensive. + + Args: + directory: Directory to scan + since_timestamp: Unix timestamp to find files newer than + + Returns: + List of relative file paths modified since the timestamp (respects .bmignore) + """ + # Convert timestamp to find-compatible format + since_date = datetime.fromtimestamp(since_timestamp).strftime("%Y-%m-%d %H:%M:%S") + + process = await asyncio.create_subprocess_shell( + f'find "{directory}" -type f -newermt "{since_date}"', + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + + if process.returncode != 0: + error_msg = stderr.decode().strip() + logger.error( + f"SCAN OPTIMIZATION FAILED: find -newermt command failed with exit code {process.returncode}, " + f"error: {error_msg}. Falling back to full scan. " + f"This will cause slow syncs on large projects!" + ) + # Track optimization failures for visibility + logfire.metric_counter("sync.scan.optimization_failure").add(1) + # Fallback to full scan + return await self._scan_directory_full(directory) + + # Convert absolute paths to relative and filter through ignore patterns + file_paths = [] + for line in stdout.decode().splitlines(): + if line: + try: + abs_path = Path(line) + rel_path = abs_path.relative_to(directory).as_posix() + + # Apply ignore patterns (same as scan_directory) + if should_ignore_path(abs_path, directory, self._ignore_patterns): + logger.trace(f"Ignoring path per .bmignore: {rel_path}") + continue + + file_paths.append(rel_path) + except ValueError: + # Path is not relative to directory, skip it + logger.warning(f"Skipping file not under directory: {line}") + continue + + return file_paths + + async def _scan_directory_full(self, directory: Path) -> List[str]: + """Full directory scan returning all file paths. + + Uses scan_directory() which respects .bmignore patterns. Args: directory: Directory to scan Returns: - ScanResult containing found files and any errors + List of relative file paths (respects .bmignore) """ - start_time = time.time() + file_paths = [] + async for file_path_str, _ in self.scan_directory(directory): + rel_path = Path(file_path_str).relative_to(directory).as_posix() + file_paths.append(rel_path) + return file_paths - logger.debug(f"Scanning directory {directory}") - result = ScanResult() - ignored_count = 0 + async def scan_directory(self, directory: Path) -> AsyncIterator[Tuple[str, os.stat_result]]: + """Stream files from directory using aiofiles.os.scandir() with cached stat info. - for root, dirnames, filenames in os.walk(str(directory)): - # Convert root to Path for easier manipulation - root_path = Path(root) + This method uses aiofiles.os.scandir() to leverage async I/O and cached stat + information from directory entries. This reduces network I/O by 50% on network + filesystems like TigrisFS by avoiding redundant stat() calls. - # Filter out ignored directories in-place - dirnames_to_remove = [] - for dirname in dirnames: - dir_path = root_path / dirname - if should_ignore_path(dir_path, directory, self._ignore_patterns): - dirnames_to_remove.append(dirname) - ignored_count += 1 + Args: + directory: Directory to scan - # Remove ignored directories from dirnames to prevent os.walk from descending - for dirname in dirnames_to_remove: - dirnames.remove(dirname) + Yields: + Tuples of (absolute_file_path, stat_info) for each file + """ + try: + entries = await aiofiles.os.scandir(directory) + except PermissionError: + logger.warning(f"Permission denied scanning directory: {directory}") + return - for filename in filenames: - path = root_path / filename + results = [] + subdirs = [] - # Check if file should be ignored - if should_ignore_path(path, directory, self._ignore_patterns): - ignored_count += 1 - logger.trace(f"Ignoring file per .bmignore: {path.relative_to(directory)}") - continue + for entry in entries: + entry_path = Path(entry.path) - rel_path = path.relative_to(directory).as_posix() - checksum = await self._compute_checksum_async(rel_path) - result.files[rel_path] = checksum - result.checksums[checksum] = rel_path + # Check ignore patterns + if should_ignore_path(entry_path, directory, self._ignore_patterns): + logger.trace(f"Ignoring path per .bmignore: {entry_path.relative_to(directory)}") + continue - logger.trace(f"Found file, path={rel_path}, checksum={checksum}") + if entry.is_dir(follow_symlinks=False): + # Collect subdirectories to recurse into + subdirs.append(entry_path) + elif entry.is_file(follow_symlinks=False): + # Get cached stat info (no extra syscall!) + stat_info = entry.stat(follow_symlinks=False) + results.append((entry.path, stat_info)) - duration_ms = int((time.time() - start_time) * 1000) - logger.debug( - f"{directory} scan completed " - f"directory={str(directory)} " - f"files_found={len(result.files)} " - f"files_ignored={ignored_count} " - f"duration_ms={duration_ms}" - ) + # Yield files from current directory + for file_path, stat_info in results: + yield (file_path, stat_info) - return result + # Recurse into subdirectories + for subdir in subdirs: + async for result in self.scan_directory(subdir): + yield result async def get_sync_service(project: Project) -> SyncService: # pragma: no cover @@ -986,6 +1196,7 @@ async def get_sync_service(project: Project) -> SyncService: # pragma: no cover observation_repository = ObservationRepository(session_maker, project_id=project.id) relation_repository = RelationRepository(session_maker, project_id=project.id) search_repository = SearchRepository(session_maker, project_id=project.id) + project_repository = ProjectRepository(session_maker) # Initialize services search_service = SearchService(search_repository, entity_repository, file_service) @@ -1008,6 +1219,7 @@ async def get_sync_service(project: Project) -> SyncService: # pragma: no cover entity_parser=entity_parser, entity_repository=entity_repository, relation_repository=relation_repository, + project_repository=project_repository, search_service=search_service, file_service=file_service, ) diff --git a/src/basic_memory/sync/watch_service.py b/src/basic_memory/sync/watch_service.py index d1ead692..03964d8e 100644 --- a/src/basic_memory/sync/watch_service.py +++ b/src/basic_memory/sync/watch_service.py @@ -239,11 +239,14 @@ class WatchService: # Check if project still exists in configuration before processing # This prevents deleted projects from being recreated by background sync from basic_memory.config import ConfigManager + config_manager = ConfigManager() - if project.name not in config_manager.projects and project.permalink not in config_manager.projects: + if ( + project.name not in config_manager.projects + and project.permalink not in config_manager.projects + ): logger.info( - f"Skipping sync for deleted project: {project.name}, " - f"change_count={len(changes)}" + f"Skipping sync for deleted project: {project.name}, change_count={len(changes)}" ) return diff --git a/src/basic_memory/utils.py b/src/basic_memory/utils.py index d73964fc..569ea588 100644 --- a/src/basic_memory/utils.py +++ b/src/basic_memory/utils.py @@ -185,6 +185,21 @@ def setup_logging( logger.info(f"ENV: '{env}' Log level: '{log_level}' Logging to {log_file}") + # Bind environment context for structured logging (works in both local and cloud) + tenant_id = os.getenv("BASIC_MEMORY_TENANT_ID", "local") + fly_app_name = os.getenv("FLY_APP_NAME", "local") + fly_machine_id = os.getenv("FLY_MACHINE_ID", "local") + fly_region = os.getenv("FLY_REGION", "local") + + logger.configure( + extra={ + "tenant_id": tenant_id, + "fly_app_name": fly_app_name, + "fly_machine_id": fly_machine_id, + "fly_region": fly_region, + } + ) + # Reduce noise from third-party libraries noisy_loggers = { # HTTP client logs diff --git a/test-int/test_disable_permalinks_integration.py b/test-int/test_disable_permalinks_integration.py index 5cee7c5f..670df70c 100644 --- a/test-int/test_disable_permalinks_integration.py +++ b/test-int/test_disable_permalinks_integration.py @@ -8,6 +8,7 @@ from basic_memory.repository import ( EntityRepository, ObservationRepository, RelationRepository, + ProjectRepository, ) from basic_memory.repository.search_repository import SearchRepository from basic_memory.schemas import Entity as EntitySchema @@ -88,6 +89,7 @@ async def test_disable_permalinks_sync_workflow(tmp_path, engine_factory): observation_repository = ObservationRepository(session_maker, project_id=1) relation_repository = RelationRepository(session_maker, project_id=1) search_repository = SearchRepository(session_maker, project_id=1) + project_repository = ProjectRepository(session_maker) # Setup services entity_parser = EntityParser(tmp_path) @@ -110,6 +112,7 @@ async def test_disable_permalinks_sync_workflow(tmp_path, engine_factory): sync_service = SyncService( app_config=app_config, entity_service=entity_service, + project_repository=project_repository, entity_parser=entity_parser, entity_repository=entity_repository, relation_repository=relation_repository, diff --git a/tests/conftest.py b/tests/conftest.py index bcd78609..d380b1f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -250,6 +250,7 @@ async def sync_service( app_config: BasicMemoryConfig, entity_service: EntityService, entity_parser: EntityParser, + project_repository: ProjectRepository, entity_repository: EntityRepository, relation_repository: RelationRepository, search_service: SearchService, @@ -259,6 +260,7 @@ async def sync_service( return SyncService( app_config=app_config, entity_service=entity_service, + project_repository=project_repository, entity_repository=entity_repository, relation_repository=relation_repository, entity_parser=entity_parser, diff --git a/tests/mcp/test_permalink_collision_file_overwrite.py b/tests/mcp/test_permalink_collision_file_overwrite.py index 3f167df9..c4b0edf7 100644 --- a/tests/mcp/test_permalink_collision_file_overwrite.py +++ b/tests/mcp/test_permalink_collision_file_overwrite.py @@ -24,6 +24,22 @@ from basic_memory.config import ProjectConfig from basic_memory.services import EntityService +async def force_full_scan(sync_service: SyncService) -> None: + """Force next sync to do a full scan by clearing watermark (for testing moves/deletions).""" + if sync_service.entity_repository.project_id is not None: + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + if project: + await sync_service.project_repository.update( + project.id, + { + "last_scan_timestamp": None, + "last_file_count": None, + }, + ) + + @pytest.mark.asyncio async def test_permalink_collision_should_not_overwrite_different_file(app, test_project): """Test that creating notes with different titles doesn't overwrite existing files. @@ -299,6 +315,10 @@ async def test_sync_permalink_collision_file_overwrite_bug( node_b_file = edge_cases_dir / "Node B.md" node_b_file.write_text(node_b_content) + # Force full scan to detect the new file + # (file just created may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Sync to create Node B await sync_service.sync(project_dir) @@ -323,6 +343,10 @@ async def test_sync_permalink_collision_file_overwrite_bug( node_c_file = edge_cases_dir / "Node C.md" node_c_file.write_text(node_c_content) + # Force full scan to detect the new file + # (file just created may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Sync to create Node C - THIS IS WHERE THE BUG OCCURS await sync_service.sync(project_dir) diff --git a/tests/mcp/test_tool_sync_status.py b/tests/mcp/test_tool_sync_status.py deleted file mode 100644 index e74c4a1f..00000000 --- a/tests/mcp/test_tool_sync_status.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Tests for sync_status MCP tool.""" - -import pytest -from unittest.mock import MagicMock, patch - -from basic_memory.mcp.tools.sync_status import sync_status -from basic_memory.services.sync_status_service import ( - SyncStatus, - ProjectSyncStatus, - SyncStatusTracker, -) - - -@pytest.mark.asyncio -async def test_sync_status_completed(): - """Test sync_status when all operations are completed.""" - # Mock sync status tracker with ready status - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = True - mock_tracker.get_summary.return_value = "✅ All projects synced successfully" - mock_tracker.get_all_projects.return_value = {} - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn() - - assert "Basic Memory Sync Status" in result - assert "System Ready**: ✅ Yes" in result - assert "All sync operations completed" in result - assert "File indexing is complete" in result - assert "knowledge base is ready for use" in result - - -@pytest.mark.asyncio -async def test_sync_status_in_progress(): - """Test sync_status when sync is in progress.""" - # Mock sync status tracker with in progress status - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = False - mock_tracker.get_summary.return_value = "🔄 Syncing 2 projects (5/10 files, 50%)" - - # Mock active projects - project1 = ProjectSyncStatus( - project_name="project1", - status=SyncStatus.SYNCING, - message="Processing new files", - files_total=5, - files_processed=3, - ) - project2 = ProjectSyncStatus( - project_name="project2", - status=SyncStatus.SCANNING, - message="Scanning files", - files_total=5, - files_processed=2, - ) - - mock_tracker.get_all_projects.return_value = {"project1": project1, "project2": project2} - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn() - - assert "Basic Memory Sync Status" in result - assert "System Ready**: 🔄 Processing" in result - assert "File synchronization in progress" in result - assert "project1**: Processing new files (3/5, 60%)" in result - assert "project2**: Scanning files (2/5, 40%)" in result - assert "Scanning and indexing markdown files" in result - assert "Use this tool again to check progress" in result - - -@pytest.mark.asyncio -async def test_sync_status_failed(): - """Test sync_status when sync has failed.""" - # Mock sync status tracker with failed project - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = False - mock_tracker.get_summary.return_value = "❌ Sync failed for: project1" - - # Mock failed project - failed_project = ProjectSyncStatus( - project_name="project1", - status=SyncStatus.FAILED, - message="Sync failed", - error="Permission denied", - ) - - mock_tracker.get_all_projects.return_value = {"project1": failed_project} - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn() - - assert "Basic Memory Sync Status" in result - assert "System Ready**: 🔄 Processing" in result - assert "Some projects failed to sync" in result - assert "project1**: Permission denied" in result - assert "Check the logs for detailed error information" in result - assert "Try restarting the MCP server" in result - - -@pytest.mark.asyncio -async def test_sync_status_idle(): - """Test sync_status when system is idle.""" - # Mock sync status tracker with idle status - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = True - mock_tracker.get_summary.return_value = "✅ System ready" - mock_tracker.get_all_projects.return_value = {} - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn() - - assert "Basic Memory Sync Status" in result - assert "System Ready**: ✅ Yes" in result - assert "All sync operations completed" in result - - -@pytest.mark.asyncio -async def test_sync_status_with_project(): - """Test sync_status with specific project context.""" - # Mock sync status tracker - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = True - mock_tracker.get_summary.return_value = "✅ All projects synced successfully" - - # Mock specific project status - project_status = ProjectSyncStatus( - project_name="test-project", - status=SyncStatus.COMPLETED, - message="Sync completed", - files_total=10, - files_processed=10, - ) - mock_tracker.get_project_status.return_value = project_status - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn(project="test-project") - - # The function should use the original logic for project-specific queries - # But since we changed the implementation, let's just verify it doesn't crash - assert "Basic Memory Sync Status" in result - - -@pytest.mark.asyncio -async def test_sync_status_pending(): - """Test sync_status when no projects are active.""" - # Mock sync status tracker with no active projects - mock_tracker = MagicMock(spec=SyncStatusTracker) - mock_tracker.is_ready = False - mock_tracker.get_summary.return_value = "✅ System ready" - mock_tracker.get_all_projects.return_value = {} - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await sync_status.fn() - - assert "Basic Memory Sync Status" in result - assert "Sync operations pending" in result - assert "usually resolves automatically" in result - - -@pytest.mark.asyncio -async def test_sync_status_error_handling(): - """Test sync_status handles errors gracefully.""" - # Mock sync status tracker that raises an exception - with patch("basic_memory.services.sync_status_service.sync_status_tracker") as mock_tracker: - mock_tracker.is_ready = True - mock_tracker.get_summary.side_effect = Exception("Test error") - - result = await sync_status.fn() - - assert "Unable to check sync status**: Test error" in result diff --git a/tests/mcp/test_tool_utils.py b/tests/mcp/test_tool_utils.py index a537e82a..e0ab67ef 100644 --- a/tests/mcp/test_tool_utils.py +++ b/tests/mcp/test_tool_utils.py @@ -1,6 +1,6 @@ """Tests for MCP tool utilities.""" -from unittest.mock import AsyncMock, patch, MagicMock +from unittest.mock import AsyncMock import pytest from httpx import AsyncClient, HTTPStatusError @@ -12,8 +12,6 @@ from basic_memory.mcp.tools.utils import ( call_put, call_delete, get_error_message, - check_migration_status, - wait_for_migration_or_return_status, ) @@ -184,82 +182,3 @@ async def test_call_post_with_json(mock_response): mock_post.assert_called_once() call_kwargs = mock_post.call_args[1] assert call_kwargs["json"] == json_data - - -class TestMigrationStatus: - """Test migration status checking functions.""" - - def test_check_migration_status_ready(self): - """Test check_migration_status when system is ready.""" - mock_tracker = MagicMock() - mock_tracker.is_ready = True - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = check_migration_status() - assert result is None - - def test_check_migration_status_not_ready(self): - """Test check_migration_status when sync is in progress.""" - mock_tracker = MagicMock() - mock_tracker.is_ready = False - mock_tracker.get_summary.return_value = "Sync in progress..." - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = check_migration_status() - assert result == "Sync in progress..." - mock_tracker.get_summary.assert_called_once() - - def test_check_migration_status_exception(self): - """Test check_migration_status with import/other exception.""" - # Mock the import itself to raise an exception - with patch("builtins.__import__", side_effect=ImportError("Module not found")): - result = check_migration_status() - assert result is None - - @pytest.mark.asyncio - async def test_wait_for_migration_ready(self): - """Test wait_for_migration when system is already ready.""" - mock_tracker = MagicMock() - mock_tracker.is_ready = True - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - result = await wait_for_migration_or_return_status() - assert result is None - - @pytest.mark.asyncio - async def test_wait_for_migration_becomes_ready(self): - """Test wait_for_migration when system becomes ready during wait.""" - mock_tracker = MagicMock() - mock_tracker.is_ready = False - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - # Mock asyncio.sleep to make tracker ready after first check - async def mock_sleep(delay): - mock_tracker.is_ready = True - - with patch("asyncio.sleep", side_effect=mock_sleep): - result = await wait_for_migration_or_return_status(timeout=1.0) - assert result is None - - @pytest.mark.asyncio - async def test_wait_for_migration_timeout(self): - """Test wait_for_migration when timeout occurs.""" - mock_tracker = MagicMock() - mock_tracker.is_ready = False - mock_tracker.get_summary.return_value = "Still syncing..." - - with patch("basic_memory.services.sync_status_service.sync_status_tracker", mock_tracker): - with patch("asyncio.sleep", new_callable=AsyncMock): - result = await wait_for_migration_or_return_status(timeout=0.1) - assert result == "Still syncing..." - mock_tracker.get_summary.assert_called_once() - - @pytest.mark.asyncio - async def test_wait_for_migration_exception(self): - """Test wait_for_migration with exception during checking.""" - with patch( - "basic_memory.services.sync_status_service.sync_status_tracker", - side_effect=Exception("Test error"), - ): - result = await wait_for_migration_or_return_status() - assert result is None diff --git a/tests/repository/test_entity_repository.py b/tests/repository/test_entity_repository.py index 9b585cae..0dedb5c0 100644 --- a/tests/repository/test_entity_repository.py +++ b/tests/repository/test_entity_repository.py @@ -644,3 +644,169 @@ async def test_find_by_directory_prefix_basic_fields_only( assert entity.entity_type == "test" assert entity.content_type == "text/markdown" assert entity.updated_at is not None + + +@pytest.mark.asyncio +async def test_get_all_file_paths(entity_repository: EntityRepository, session_maker): + """Test getting all file paths for deletion detection during sync.""" + # Create test entities with various file paths + async with db.scoped_session(session_maker) as session: + entities = [ + Entity( + project_id=entity_repository.project_id, + title="File 1", + entity_type="test", + permalink="docs/file1", + file_path="docs/file1.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Entity( + project_id=entity_repository.project_id, + title="File 2", + entity_type="test", + permalink="specs/file2", + file_path="specs/file2.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Entity( + project_id=entity_repository.project_id, + title="File 3", + entity_type="test", + permalink="notes/file3", + file_path="notes/file3.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + ] + session.add_all(entities) + await session.flush() + + # Get all file paths + file_paths = await entity_repository.get_all_file_paths() + + # Verify results + assert isinstance(file_paths, list) + assert len(file_paths) == 3 + assert set(file_paths) == {"docs/file1.md", "specs/file2.md", "notes/file3.md"} + + +@pytest.mark.asyncio +async def test_get_all_file_paths_empty_db(entity_repository: EntityRepository): + """Test getting all file paths when database is empty.""" + file_paths = await entity_repository.get_all_file_paths() + assert file_paths == [] + + +@pytest.mark.asyncio +async def test_get_all_file_paths_performance(entity_repository: EntityRepository, session_maker): + """Test that get_all_file_paths doesn't load entities or relationships. + + This method is optimized for deletion detection during streaming sync. + It should only query file_path strings, not full entity objects. + """ + # Create test entity with observations and relations + async with db.scoped_session(session_maker) as session: + # Create entities + entity1 = Entity( + project_id=entity_repository.project_id, + title="Entity 1", + entity_type="test", + permalink="test/entity1", + file_path="test/entity1.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + entity2 = Entity( + project_id=entity_repository.project_id, + title="Entity 2", + entity_type="test", + permalink="test/entity2", + file_path="test/entity2.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + session.add_all([entity1, entity2]) + await session.flush() + + # Add observations to entity1 + observation = Observation( + entity_id=entity1.id, + content="Test observation", + category="note", + ) + session.add(observation) + + # Add relation between entities + relation = Relation( + from_id=entity1.id, + to_id=entity2.id, + to_name=entity2.title, + relation_type="relates_to", + ) + session.add(relation) + await session.flush() + + # Get all file paths - should be fast and not load relationships + file_paths = await entity_repository.get_all_file_paths() + + # Verify results - just file paths, no entities or relationships loaded + assert len(file_paths) == 2 + assert set(file_paths) == {"test/entity1.md", "test/entity2.md"} + + # Result should be list of strings, not entity objects + for path in file_paths: + assert isinstance(path, str) + + +@pytest.mark.asyncio +async def test_get_all_file_paths_project_isolation( + entity_repository: EntityRepository, session_maker +): + """Test that get_all_file_paths only returns paths from the current project.""" + # Create entities in the repository's project + async with db.scoped_session(session_maker) as session: + entity1 = Entity( + project_id=entity_repository.project_id, + title="Project 1 File", + entity_type="test", + permalink="test/file1", + file_path="test/file1.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + session.add(entity1) + await session.flush() + + # Create a second project + project2 = Project(name="other-project", path="/tmp/other") + session.add(project2) + await session.flush() + + # Create entity in different project + entity2 = Entity( + project_id=project2.id, + title="Project 2 File", + entity_type="test", + permalink="test/file2", + file_path="test/file2.md", + content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + session.add(entity2) + await session.flush() + + # Get all file paths for project 1 + file_paths = await entity_repository.get_all_file_paths() + + # Should only include files from project 1 + assert len(file_paths) == 1 + assert file_paths == ["test/file1.md"] diff --git a/tests/services/test_project_service.py b/tests/services/test_project_service.py index 77d19154..8e564346 100644 --- a/tests/services/test_project_service.py +++ b/tests/services/test_project_service.py @@ -1225,7 +1225,7 @@ async def test_synchronize_projects_removes_db_only_projects(project_service: Pr "permalink": test_project_name.lower().replace(" ", "-"), "is_active": True, } - created_project = await project_service.repository.create(project_data) + await project_service.repository.create(project_data) # Verify it exists in DB but not in config db_project = await project_service.repository.get_by_name(test_project_name) diff --git a/tests/services/test_sync_status_service.py b/tests/services/test_sync_status_service.py deleted file mode 100644 index 00806784..00000000 --- a/tests/services/test_sync_status_service.py +++ /dev/null @@ -1,262 +0,0 @@ -"""Test sync status service functionality.""" - -import pytest -from basic_memory.services.sync_status_service import SyncStatusTracker, SyncStatus - - -@pytest.fixture -def sync_tracker(): - """Create a fresh sync status tracker for testing.""" - return SyncStatusTracker() - - -def test_sync_tracker_initial_state(sync_tracker): - """Test initial state of sync tracker.""" - assert sync_tracker.is_ready - assert not sync_tracker.is_syncing - assert sync_tracker.global_status == SyncStatus.IDLE - assert sync_tracker.get_summary() == "✅ System ready" - - # Test project-specific ready check for unknown project - assert sync_tracker.is_project_ready("unknown-project") - - -def test_start_project_sync(sync_tracker): - """Test starting project sync.""" - sync_tracker.start_project_sync("test-project", files_total=10) - - assert not sync_tracker.is_ready - assert sync_tracker.is_syncing - assert sync_tracker.global_status == SyncStatus.SYNCING - - project_status = sync_tracker.get_project_status("test-project") - assert project_status is not None - assert project_status.status == SyncStatus.SCANNING - assert project_status.message == "Scanning files" - assert project_status.files_total == 10 - - -def test_update_project_progress(sync_tracker): - """Test updating project progress.""" - sync_tracker.start_project_sync("test-project") # Use default files_total=0 - sync_tracker.update_project_progress( - "test-project", SyncStatus.SYNCING, "Processing files", files_processed=5, files_total=10 - ) - - project_status = sync_tracker.get_project_status("test-project") - assert project_status.status == SyncStatus.SYNCING - assert project_status.message == "Processing files" - assert project_status.files_processed == 5 - assert project_status.files_total == 10 - assert sync_tracker.global_status == SyncStatus.SYNCING - - -def test_complete_project_sync(sync_tracker): - """Test completing project sync.""" - sync_tracker.start_project_sync("test-project") - sync_tracker.complete_project_sync("test-project") - - assert sync_tracker.is_ready - assert not sync_tracker.is_syncing - assert sync_tracker.global_status == SyncStatus.COMPLETED - - project_status = sync_tracker.get_project_status("test-project") - assert project_status.status == SyncStatus.COMPLETED - assert project_status.message == "Sync completed" - - -def test_fail_project_sync(sync_tracker): - """Test failing project sync.""" - sync_tracker.start_project_sync("test-project") - sync_tracker.fail_project_sync("test-project", "Connection error") - - assert not sync_tracker.is_ready - assert not sync_tracker.is_syncing - assert sync_tracker.global_status == SyncStatus.FAILED - - project_status = sync_tracker.get_project_status("test-project") - assert project_status.status == SyncStatus.FAILED - assert project_status.error == "Connection error" - - -def test_start_project_watch(sync_tracker): - """Test starting project watch mode.""" - sync_tracker.start_project_watch("test-project") - - assert sync_tracker.is_ready - assert not sync_tracker.is_syncing - assert sync_tracker.global_status == SyncStatus.COMPLETED - - project_status = sync_tracker.get_project_status("test-project") - assert project_status.status == SyncStatus.WATCHING - assert project_status.message == "Watching for changes" - - -def test_multiple_projects_status(sync_tracker): - """Test status with multiple projects.""" - sync_tracker.start_project_sync("project1") - sync_tracker.start_project_sync("project2") - - # Both scanning - should be syncing - assert sync_tracker.global_status == SyncStatus.SYNCING - assert sync_tracker.is_syncing - - # Complete one project - sync_tracker.complete_project_sync("project1") - assert sync_tracker.global_status == SyncStatus.SYNCING # Still syncing - - # Complete second project - sync_tracker.complete_project_sync("project2") - assert sync_tracker.global_status == SyncStatus.COMPLETED - assert sync_tracker.is_ready - - -def test_mixed_project_statuses(sync_tracker): - """Test mixed project statuses.""" - sync_tracker.start_project_sync("project1") - sync_tracker.start_project_sync("project2") - - # Fail one project - sync_tracker.fail_project_sync("project1", "Error") - # Complete other project - sync_tracker.complete_project_sync("project2") - - # Should show failed status - assert sync_tracker.global_status == SyncStatus.FAILED - assert not sync_tracker.is_ready - - -def test_get_summary_with_progress(sync_tracker): - """Test summary with progress information.""" - sync_tracker.start_project_sync("project1") - sync_tracker.update_project_progress( - "project1", SyncStatus.SYNCING, "Processing", files_processed=25, files_total=100 - ) - - summary = sync_tracker.get_summary() - assert "🔄 Syncing 1 projects" in summary - assert "(25/100 files, 25%)" in summary - - -def test_get_all_projects(sync_tracker): - """Test getting all project statuses.""" - sync_tracker.start_project_sync("project1") - sync_tracker.start_project_sync("project2") - - all_projects = sync_tracker.get_all_projects() - assert len(all_projects) == 2 - assert "project1" in all_projects - assert "project2" in all_projects - assert all_projects["project1"].status == SyncStatus.SCANNING - assert all_projects["project2"].status == SyncStatus.SCANNING - - -def test_clear_completed(sync_tracker): - """Test clearing completed project statuses.""" - sync_tracker.start_project_sync("project1") - sync_tracker.start_project_sync("project2") - - sync_tracker.complete_project_sync("project1") - sync_tracker.fail_project_sync("project2", "Error") - - # Should have 2 projects before clearing - assert len(sync_tracker.get_all_projects()) == 2 - - sync_tracker.clear_completed() - - # Should only have the failed project after clearing - remaining = sync_tracker.get_all_projects() - assert len(remaining) == 1 - assert "project2" in remaining - assert remaining["project2"].status == SyncStatus.FAILED - - -def test_summary_messages(sync_tracker): - """Test various summary messages.""" - # Initial state - assert sync_tracker.get_summary() == "✅ System ready" - - # All completed - sync_tracker.start_project_sync("project1") - sync_tracker.complete_project_sync("project1") - assert sync_tracker.get_summary() == "✅ All projects synced successfully" - - # Failed projects - sync_tracker.fail_project_sync("project1", "Test error") - assert "❌ Sync failed for: project1" in sync_tracker.get_summary() - - -def test_global_status_edge_cases(sync_tracker): - """Test edge cases for global status calculation.""" - # Test mixed statuses (some completed, some watching) - should be completed - sync_tracker.start_project_sync("project1") - sync_tracker.start_project_sync("project2") - - sync_tracker.complete_project_sync("project1") - sync_tracker.start_project_watch("project2") - - assert sync_tracker.global_status == SyncStatus.COMPLETED - - # Test fallback case - create a scenario that doesn't match specific conditions - sync_tracker.start_project_sync("project3") - sync_tracker.update_project_progress("project3", SyncStatus.IDLE, "Idle") - - # This should trigger the "else" clause in _update_global_status - assert sync_tracker.global_status == SyncStatus.SYNCING - - -def test_summary_without_file_counts(sync_tracker): - """Test summary when projects don't have file counts.""" - sync_tracker.start_project_sync("project1") # files_total defaults to 0 - sync_tracker.start_project_sync("project2") # files_total defaults to 0 - - # Don't set file counts - should use the fallback message - summary = sync_tracker.get_summary() - assert "🔄 Syncing 2 projects" in summary - assert "files" not in summary # Should not show file progress - - -def test_is_project_ready_functionality(sync_tracker): - """Test project-specific ready checks.""" - # Unknown project should be ready - assert sync_tracker.is_project_ready("unknown-project") - - # Project in different states - sync_tracker.start_project_sync("scanning-project") - assert not sync_tracker.is_project_ready("scanning-project") # SCANNING = not ready - - sync_tracker.update_project_progress("scanning-project", SyncStatus.SYNCING, "Processing") - assert not sync_tracker.is_project_ready("scanning-project") # SYNCING = not ready - - sync_tracker.fail_project_sync("scanning-project", "Test error") - assert not sync_tracker.is_project_ready("scanning-project") # FAILED = not ready - - sync_tracker.complete_project_sync("scanning-project") - assert sync_tracker.is_project_ready("scanning-project") # COMPLETED = ready - - # Test watching project - sync_tracker.start_project_watch("watching-project") - assert sync_tracker.is_project_ready("watching-project") # WATCHING = ready - - -def test_project_isolation_scenario(sync_tracker): - """Test the specific bug scenario: project isolation with mixed sync states.""" - # Set up the bug scenario: one failed project, one healthy project - sync_tracker.start_project_sync("main") - sync_tracker.fail_project_sync( - "main", "UNIQUE constraint failed: entity.file_path, entity.project_id" - ) - - sync_tracker.start_project_sync("basic-memory-testing-20250626-1009") - sync_tracker.complete_project_sync("basic-memory-testing-20250626-1009") - sync_tracker.start_project_watch("basic-memory-testing-20250626-1009") - - # Global status should be failed due to "main" project - assert sync_tracker.global_status == SyncStatus.FAILED - assert not sync_tracker.is_ready - - # But the healthy project should be ready for operations - assert sync_tracker.is_project_ready("basic-memory-testing-20250626-1009") - assert not sync_tracker.is_project_ready("main") - - # This demonstrates the fix: project-specific checks allow isolation diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index 98f5d8d6..6a620cfc 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -23,6 +23,32 @@ async def create_test_file(path: Path, content: str = "test content") -> None: path.write_text(content) +async def touch_file(path: Path) -> None: + """Touch a file to update its mtime (for watermark testing).""" + import time + + # Read and rewrite to update mtime + content = path.read_text() + time.sleep(0.5) # Ensure mtime changes and is newer than watermark (500ms) + path.write_text(content) + + +async def force_full_scan(sync_service: SyncService) -> None: + """Force next sync to do a full scan by clearing watermark (for testing moves/deletions).""" + if sync_service.entity_repository.project_id is not None: + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + if project: + await sync_service.project_repository.update( + project.id, + { + "last_scan_timestamp": None, + "last_file_count": None, + }, + ) + + @pytest.mark.asyncio async def test_forward_reference_resolution( sync_service: SyncService, @@ -62,7 +88,12 @@ type: knowledge # Target Doc Target content """ - await create_test_file(project_dir / "target_doc.md", target_content) + target_file = project_dir / "target_doc.md" + await create_test_file(target_file, target_content) + + # Force full scan to ensure the new file is detected + # Incremental scans have timing precision issues with watermarks on some filesystems + await force_full_scan(sync_service) # Sync again - should resolve the reference await sync_service.sync(project_config.home) @@ -475,7 +506,7 @@ async def test_sync_empty_directories(sync_service: SyncService, project_config: await sync_service.sync(project_config.home) # Should not raise exceptions for empty dirs - assert (project_config.home).exists() + assert project_config.home.exists() @pytest.mark.skip("flaky on Windows due to filesystem timing precision") @@ -538,8 +569,7 @@ async def test_permalink_formatting( } # Create test files - for filename, _ in test_files.items(): - content: str = """ + content: str = """ --- type: knowledge created: 2024-01-01 @@ -549,10 +579,11 @@ modified: 2024-01-01 Testing permalink generation. """ + for filename, _ in test_files.items(): await create_test_file(project_config.home / filename, content) - # Run sync - await sync_service.sync(project_config.home) + # Run sync once after all files are created + await sync_service.sync(project_config.home) # Verify permalinks entities = await entity_service.repository.find_all() @@ -568,7 +599,6 @@ Testing permalink generation. async def test_handle_entity_deletion( test_graph, sync_service: SyncService, - project_config: ProjectConfig, entity_repository: EntityRepository, search_service: SearchService, ): @@ -658,7 +688,6 @@ async def test_sync_updates_timestamps_on_file_modification( not the database operation time. This is critical for accurate temporal ordering in search and recent_activity queries. """ - import time project_dir = project_config.home @@ -680,10 +709,7 @@ Initial content for timestamp test entity_before = await entity_service.get_by_permalink("timestamp-test") initial_updated_at = entity_before.updated_at - # Wait a bit to ensure filesystem timestamp changes - time.sleep(0.1) - - # Modify the file content + # Modify the file content and update mtime to be newer than watermark modified_content = """ --- type: knowledge @@ -696,12 +722,17 @@ Modified content for timestamp test """ file_path.write_text(modified_content) - # Wait to ensure mtime is different - time.sleep(0.1) + # Touch file to ensure mtime is newer than watermark + # This uses our helper which sleeps 500ms and rewrites to guarantee mtime change + await touch_file(file_path) # Get the file's modification time after our changes file_stats_after_modification = file_path.stat() + # Force full scan to ensure the modified file is detected + # (incremental scans have timing precision issues with watermarks on some filesystems) + await force_full_scan(sync_service) + # Re-sync the modified file await sync_service.sync(project_config.home) @@ -758,7 +789,11 @@ Content for move test new_path.parent.mkdir(parents=True) old_path.rename(new_path) - # Sync again + # Force full scan to detect the move + # (rename doesn't update mtime, so incremental scan won't find it) + await force_full_scan(sync_service) + + # Second sync should detect the move await sync_service.sync(project_config.home) # Check search index has updated path @@ -772,7 +807,6 @@ async def test_sync_null_checksum_cleanup( sync_service: SyncService, project_config: ProjectConfig, entity_service: EntityService, - app_config, ): """Test handling of entities with null checksums from incomplete syncs.""" # Create entity with null checksum (simulating incomplete sync) @@ -838,6 +872,10 @@ Content for move test new_path.parent.mkdir(parents=True) old_path.rename(new_path) + # Force full scan to detect the move + # (rename doesn't update mtime, so incremental scan won't find it) + await force_full_scan(sync_service) + # Sync again await sync_service.sync(project_config.home) @@ -857,6 +895,10 @@ Content for move test old_path.parent.mkdir(parents=True, exist_ok=True) await create_test_file(old_path, content) + # Force full scan to detect the new file + # (file just created may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Sync new file await sync_service.sync(project_config.home) @@ -922,6 +964,10 @@ test content """ two_file.write_text(updated_content) + # Force full scan to detect the modified file + # (file just modified may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Run sync await sync_service.sync(project_config.home) @@ -943,6 +989,10 @@ test content new_file = project_dir / "new.md" await create_test_file(new_file, new_content) + # Force full scan to detect the new file + # (file just created may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Run another time await sync_service.sync(project_config.home) @@ -987,7 +1037,6 @@ async def test_sync_permalink_updated_on_move( ): """Test that we update a permalink on a file move if set in config .""" project_dir = project_config.home - sync_service.project_config = project_config # Create initial file content = dedent( @@ -1016,6 +1065,10 @@ async def test_sync_permalink_updated_on_move( new_path.parent.mkdir(parents=True) old_path.rename(new_path) + # Force full scan to detect the move + # (rename doesn't update mtime, so incremental scan won't find it) + await force_full_scan(sync_service) + # Sync again await sync_service.sync(project_config.home) @@ -1059,6 +1112,10 @@ async def test_sync_non_markdown_files_modified( test_files["pdf"].write_text("New content") test_files["image"].write_text("New content") + # Force full scan to detect the modified files + # (files just modified may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + report = await sync_service.sync(project_config.home) assert len(report.modified) == 2 @@ -1085,6 +1142,11 @@ async def test_sync_non_markdown_files_move(sync_service, project_config, test_f assert test_files["image"].name in [f for f in report.new] test_files["pdf"].rename(project_config.home / "moved_pdf.pdf") + + # Force full scan to detect the move + # (rename doesn't update mtime, so incremental scan won't find it) + await force_full_scan(sync_service) + report2 = await sync_service.sync(project_config.home) assert len(report2.moves) == 1 @@ -1211,7 +1273,7 @@ This is a test file for race condition handling. call_count += 1 if call_count == 1: # Simulate race condition - another process created the entity - raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) + raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) # pyright: ignore [reportArgumentType] else: return await original_add(*args, **kwargs) @@ -1293,7 +1355,7 @@ This is a test file for integrity error handling. # Mock the entity_repository.add to raise a different IntegrityError (not file_path constraint) async def mock_add(*args, **kwargs): # Simulate a different constraint violation - raise IntegrityError("UNIQUE constraint failed: entity.some_other_field", None, None) + raise IntegrityError("UNIQUE constraint failed: entity.some_other_field", None, None) # pyright: ignore [reportArgumentType] with patch.object(sync_service.entity_repository, "add", side_effect=mock_add): # Should re-raise the IntegrityError since it's not a file_path constraint @@ -1326,7 +1388,7 @@ This is a test file for entity not found after constraint violation. # Mock the entity_repository.add to raise IntegrityError async def mock_add(*args, **kwargs): - raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) + raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) # pyright: ignore [reportArgumentType] # Mock get_by_file_path to return None (entity not found) async def mock_get_by_file_path(file_path): @@ -1367,7 +1429,7 @@ This is a test file for update failure after constraint violation. # Mock the entity_repository.add to raise IntegrityError async def mock_add(*args, **kwargs): - raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) + raise IntegrityError("UNIQUE constraint failed: entity.file_path", None, None) # pyright: ignore [reportArgumentType] # Mock get_by_file_path to return an existing entity async def mock_get_by_file_path(file_path): @@ -1425,10 +1487,24 @@ async def test_circuit_breaker_skips_after_three_failures( report1 = await sync_service.sync(project_dir) assert len(report1.skipped_files) == 0 # Not skipped yet + # Touch file to trigger incremental scan + await touch_file(test_file) + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + # Second sync - should fail and record (2/3) report2 = await sync_service.sync(project_dir) assert len(report2.skipped_files) == 0 # Still not skipped + # Touch file to trigger incremental scan + await touch_file(test_file) + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + # Third sync - should fail, record (3/3), and be added to skipped list report3 = await sync_service.sync(project_dir) assert len(report3.skipped_files) == 1 @@ -1436,6 +1512,13 @@ async def test_circuit_breaker_skips_after_three_failures( assert report3.skipped_files[0].failure_count == 3 assert "Simulated sync failure" in report3.skipped_files[0].reason + # Touch file to trigger incremental scan + await touch_file(test_file) + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + # Fourth sync - should be skipped immediately without attempting report4 = await sync_service.sync(project_dir) assert len(report4.skipped_files) == 1 # Still skipped @@ -1465,7 +1548,19 @@ async def test_circuit_breaker_resets_on_file_change( with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file): # Fail 3 times to hit circuit breaker threshold await sync_service.sync(project_dir) # Fail 1 + await touch_file(test_file) # Touch to trigger incremental scan + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + await sync_service.sync(project_dir) # Fail 2 + await touch_file(test_file) # Touch to trigger incremental scan + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + report3 = await sync_service.sync(project_dir) # Fail 3 - now skipped assert len(report3.skipped_files) == 1 @@ -1482,6 +1577,10 @@ async def test_circuit_breaker_resets_on_file_change( ).strip() await create_test_file(test_file, valid_content) + # Force full scan to detect the modified file + # (file just modified may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Circuit breaker should reset and allow retry report = await sync_service.sync(project_dir) assert len(report.skipped_files) == 0 # Should not be skipped anymore @@ -1529,7 +1628,19 @@ async def test_circuit_breaker_clears_on_success( # Patch and fail twice with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file): await sync_service.sync(project_dir) # Fail 1 + await touch_file(test_file) # Touch to trigger incremental scan + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + await sync_service.sync(project_dir) # Fail 2 + await touch_file(test_file) # Touch to trigger incremental scan + + # Force full scan to ensure file is detected + # (touch may not update mtime sufficiently on all filesystems) + await force_full_scan(sync_service) + await sync_service.sync(project_dir) # Succeed # Verify failure history was cleared @@ -1593,7 +1704,11 @@ Content 3 with patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file): # Fail 3 times for file1 and file2 (file3 succeeds each time) await sync_service.sync(project_dir) # Fail count: file1=1, file2=1 + await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan + await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan await sync_service.sync(project_dir) # Fail count: file1=2, file2=2 + await touch_file(project_dir / "file1.md") # Touch to trigger incremental scan + await touch_file(project_dir / "file2.md") # Touch to trigger incremental scan report3 = await sync_service.sync(project_dir) # Fail count: file1=3, file2=3, now skipped # Both files should be skipped on third sync @@ -1622,7 +1737,7 @@ async def test_circuit_breaker_handles_checksum_computation_failure( raise ValueError("Sync failure") # Mock checksum computation to fail only during _record_failure (not during scan) - original_compute_checksum = sync_service._compute_checksum_async + original_compute_checksum = sync_service.file_service.compute_checksum call_count = 0 async def mock_compute_checksum(path): @@ -1637,8 +1752,8 @@ async def test_circuit_breaker_handles_checksum_computation_failure( with ( patch.object(sync_service, "sync_markdown_file", side_effect=mock_sync_markdown_file), patch.object( - sync_service, - "_compute_checksum_async", + sync_service.file_service, + "compute_checksum", side_effect=mock_compute_checksum, ), ): @@ -1726,3 +1841,203 @@ async def test_sync_fatal_error_terminates_sync_immediately( # Verify that no other files were attempted (sync terminated on first error) # If circuit breaker was used, we'd see file1 in failures # If sync continued, we'd see attempts for file2 and file3 + + +@pytest.mark.asyncio +async def test_scan_directory_basic(sync_service: SyncService, project_config: ProjectConfig): + """Test basic streaming directory scan functionality.""" + project_dir = project_config.home + + # Create test files in different directories + await create_test_file(project_dir / "root.md", "root content") + await create_test_file(project_dir / "subdir/file1.md", "file 1 content") + await create_test_file(project_dir / "subdir/file2.md", "file 2 content") + await create_test_file(project_dir / "subdir/nested/file3.md", "file 3 content") + + # Collect results from streaming iterator + results = [] + async for file_path, stat_info in sync_service.scan_directory(project_dir): + rel_path = Path(file_path).relative_to(project_dir).as_posix() + results.append((rel_path, stat_info)) + + # Verify all files were found + file_paths = {rel_path for rel_path, _ in results} + assert "root.md" in file_paths + assert "subdir/file1.md" in file_paths + assert "subdir/file2.md" in file_paths + assert "subdir/nested/file3.md" in file_paths + assert len(file_paths) == 4 + + # Verify stat info is present for each file + for rel_path, stat_info in results: + assert stat_info is not None + assert stat_info.st_size > 0 # Files have content + assert stat_info.st_mtime > 0 # Have modification time + + +@pytest.mark.asyncio +async def test_scan_directory_respects_ignore_patterns( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that streaming scan respects .gitignore patterns.""" + project_dir = project_config.home + + # Create .gitignore file in project (will be used along with .bmignore) + (project_dir / ".gitignore").write_text("*.ignored\n.hidden/\n") + + # Reload ignore patterns using project's .gitignore + from basic_memory.ignore_utils import load_gitignore_patterns + + sync_service._ignore_patterns = load_gitignore_patterns(project_dir) + + # Create test files - some should be ignored + await create_test_file(project_dir / "included.md", "included") + await create_test_file(project_dir / "excluded.ignored", "excluded") + await create_test_file(project_dir / ".hidden/secret.md", "secret") + await create_test_file(project_dir / "subdir/file.md", "file") + + # Collect results + results = [] + async for file_path, stat_info in sync_service.scan_directory(project_dir): + rel_path = Path(file_path).relative_to(project_dir).as_posix() + results.append(rel_path) + + # Verify ignored files were not returned + assert "included.md" in results + assert "subdir/file.md" in results + assert "excluded.ignored" not in results + assert ".hidden/secret.md" not in results + assert ".bmignore" not in results # .bmignore itself should be ignored + + +@pytest.mark.asyncio +async def test_scan_directory_cached_stat_info( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that streaming scan provides cached stat info (no redundant stat calls).""" + project_dir = project_config.home + + # Create test file + test_file = project_dir / "test.md" + await create_test_file(test_file, "test content") + + # Get stat info from streaming scan + async for file_path, stat_info in sync_service.scan_directory(project_dir): + if Path(file_path).name == "test.md": + # Get independent stat for comparison + independent_stat = test_file.stat() + + # Verify stat info matches (cached stat should be accurate) + assert stat_info.st_size == independent_stat.st_size + assert abs(stat_info.st_mtime - independent_stat.st_mtime) < 1 # Allow 1s tolerance + assert abs(stat_info.st_ctime - independent_stat.st_ctime) < 1 + break + + +@pytest.mark.asyncio +async def test_scan_directory_empty_directory( + sync_service: SyncService, project_config: ProjectConfig +): + """Test streaming scan on empty directory (ignoring hidden files).""" + project_dir = project_config.home + + # Directory exists but has no user files (may have .basic-memory config dir) + assert project_dir.exists() + + # Don't create any user files - just scan empty directory + # Scan should yield no results (hidden files are ignored by default) + results = [] + async for file_path, stat_info in sync_service.scan_directory(project_dir): + results.append(file_path) + + # Should find no files (config dirs are hidden and ignored) + assert len(results) == 0 + + +@pytest.mark.asyncio +async def test_scan_directory_handles_permission_error( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that streaming scan handles permission errors gracefully.""" + import sys + + # Skip on Windows - permission handling is different + if sys.platform == "win32": + pytest.skip("Permission tests not reliable on Windows") + + project_dir = project_config.home + + # Create accessible file + await create_test_file(project_dir / "accessible.md", "accessible") + + # Create restricted directory + restricted_dir = project_dir / "restricted" + restricted_dir.mkdir() + await create_test_file(restricted_dir / "secret.md", "secret") + + # Remove read permission from restricted directory + restricted_dir.chmod(0o000) + + try: + # Scan should handle permission error and continue + results = [] + async for file_path, stat_info in sync_service.scan_directory(project_dir): + rel_path = Path(file_path).relative_to(project_dir).as_posix() + results.append(rel_path) + + # Should have found accessible file but not restricted one + assert "accessible.md" in results + assert "restricted/secret.md" not in results + + finally: + # Restore permissions for cleanup + restricted_dir.chmod(0o755) + + +@pytest.mark.asyncio +async def test_scan_directory_non_markdown_files( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that streaming scan finds all file types, not just markdown.""" + project_dir = project_config.home + + # Create various file types + await create_test_file(project_dir / "doc.md", "markdown") + (project_dir / "image.png").write_bytes(b"PNG content") + (project_dir / "data.json").write_text('{"key": "value"}') + (project_dir / "script.py").write_text("print('hello')") + + # Collect results + results = [] + async for file_path, stat_info in sync_service.scan_directory(project_dir): + rel_path = Path(file_path).relative_to(project_dir).as_posix() + results.append(rel_path) + + # All files should be found + assert "doc.md" in results + assert "image.png" in results + assert "data.json" in results + assert "script.py" in results + + +@pytest.mark.asyncio +async def test_file_service_checksum_correctness( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that FileService computes correct checksums.""" + import hashlib + + project_dir = project_config.home + + # Test small markdown file + small_content = "Test content for checksum validation" * 10 + small_file = project_dir / "small.md" + await create_test_file(small_file, small_content) + + rel_path = small_file.relative_to(project_dir).as_posix() + checksum = await sync_service.file_service.compute_checksum(rel_path) + + # Verify checksum is correct + expected = hashlib.sha256(small_content.encode("utf-8")).hexdigest() + assert checksum == expected + assert len(checksum) == 64 # SHA256 hex digest length diff --git a/tests/sync/test_sync_service_incremental.py b/tests/sync/test_sync_service_incremental.py new file mode 100644 index 00000000..35d90d5a --- /dev/null +++ b/tests/sync/test_sync_service_incremental.py @@ -0,0 +1,729 @@ +"""Tests for incremental scan watermark optimization (Phase 1.5). + +These tests verify the scan watermark feature that dramatically improves sync +performance on large projects by: +- Using find -newermt for incremental scans (only changed files) +- Tracking last_scan_timestamp and last_file_count +- Falling back to full scan when deletions detected + +Expected performance improvements: +- No changes: 225x faster (2s vs 450s for 1,460 files) +- Few changes: 84x faster (5s vs 420s) +""" + +import time +from pathlib import Path +from textwrap import dedent + +import pytest + +from basic_memory.config import ProjectConfig +from basic_memory.sync.sync_service import SyncService + + +async def create_test_file(path: Path, content: str = "test content") -> None: + """Create a test file with given content.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +async def sleep_past_watermark(duration: float = 1.1) -> None: + """Sleep long enough to ensure mtime is newer than watermark. + + Args: + duration: Sleep duration in seconds (default 1.1s for filesystem precision) + """ + time.sleep(duration) + + +# ============================================================================== +# Scan Strategy Selection Tests +# ============================================================================== + + +@pytest.mark.asyncio +async def test_first_sync_uses_full_scan(sync_service: SyncService, project_config: ProjectConfig): + """Test that first sync (no watermark) triggers full scan.""" + project_dir = project_config.home + + # Create test files + await create_test_file(project_dir / "file1.md", "# File 1\nContent 1") + await create_test_file(project_dir / "file2.md", "# File 2\nContent 2") + + # First sync - should use full scan (no watermark exists) + report = await sync_service.sync(project_dir) + + assert len(report.new) == 2 + assert "file1.md" in report.new + assert "file2.md" in report.new + + # Verify watermark was set + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + assert project.last_scan_timestamp is not None + assert project.last_file_count >= 2 # May include config files + + +@pytest.mark.asyncio +async def test_file_count_decreased_triggers_full_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that file deletion (count decreased) triggers full scan.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + await create_test_file(project_dir / "file3.md", "# File 3") + + # First sync + await sync_service.sync(project_dir) + + # Delete a file + (project_dir / "file2.md").unlink() + + # Sleep to ensure file operations complete + await sleep_past_watermark() + + # Second sync - should detect deletion via full scan (file count decreased) + report = await sync_service.sync(project_dir) + + assert len(report.deleted) == 1 + assert "file2.md" in report.deleted + + +@pytest.mark.asyncio +async def test_file_count_same_uses_incremental_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that same file count uses incremental scan.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1\nOriginal") + await create_test_file(project_dir / "file2.md", "# File 2\nOriginal") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer than watermark + await sleep_past_watermark() + + # Modify one file (file count stays the same) + await create_test_file(project_dir / "file1.md", "# File 1\nModified") + + # Second sync - should use incremental scan (same file count) + report = await sync_service.sync(project_dir) + + assert len(report.modified) == 1 + assert "file1.md" in report.modified + + +@pytest.mark.asyncio +async def test_file_count_increased_uses_incremental_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that increased file count still uses incremental scan.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer than watermark + await sleep_past_watermark() + + # Add a new file (file count increased) + await create_test_file(project_dir / "file3.md", "# File 3") + + # Second sync - should use incremental scan and detect new file + report = await sync_service.sync(project_dir) + + assert len(report.new) == 1 + assert "file3.md" in report.new + + +# ============================================================================== +# Incremental Scan Base Cases +# ============================================================================== + + +@pytest.mark.asyncio +async def test_incremental_scan_no_changes( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan with no changes returns empty report.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure time passes + await sleep_past_watermark() + + # Second sync - no changes + report = await sync_service.sync(project_dir) + + assert len(report.new) == 0 + assert len(report.modified) == 0 + assert len(report.deleted) == 0 + assert len(report.moves) == 0 + + +@pytest.mark.asyncio +async def test_incremental_scan_detects_new_file( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan detects newly created files.""" + project_dir = project_config.home + + # Create initial file + await create_test_file(project_dir / "file1.md", "# File 1") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer than watermark + await sleep_past_watermark() + + # Create new file + await create_test_file(project_dir / "file2.md", "# File 2") + + # Second sync - should detect new file via incremental scan + report = await sync_service.sync(project_dir) + + assert len(report.new) == 1 + assert "file2.md" in report.new + assert len(report.modified) == 0 + + +@pytest.mark.asyncio +async def test_incremental_scan_detects_modified_file( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan detects modified files.""" + project_dir = project_config.home + + # Create initial files + file_path = project_dir / "file1.md" + await create_test_file(file_path, "# File 1\nOriginal content") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer than watermark + await sleep_past_watermark() + + # Modify the file + await create_test_file(file_path, "# File 1\nModified content") + + # Second sync - should detect modification via incremental scan + report = await sync_service.sync(project_dir) + + assert len(report.modified) == 1 + assert "file1.md" in report.modified + assert len(report.new) == 0 + + +@pytest.mark.asyncio +async def test_incremental_scan_detects_multiple_changes( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan detects multiple file changes.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1\nOriginal") + await create_test_file(project_dir / "file2.md", "# File 2\nOriginal") + await create_test_file(project_dir / "file3.md", "# File 3\nOriginal") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer than watermark + await sleep_past_watermark() + + # Modify multiple files + await create_test_file(project_dir / "file1.md", "# File 1\nModified") + await create_test_file(project_dir / "file3.md", "# File 3\nModified") + await create_test_file(project_dir / "file4.md", "# File 4\nNew") + + # Second sync - should detect all changes via incremental scan + report = await sync_service.sync(project_dir) + + assert len(report.modified) == 2 + assert "file1.md" in report.modified + assert "file3.md" in report.modified + assert len(report.new) == 1 + assert "file4.md" in report.new + + +# ============================================================================== +# Deletion Detection Tests +# ============================================================================== + + +@pytest.mark.asyncio +async def test_deletion_triggers_full_scan_single_file( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that deleting a single file triggers full scan.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + await create_test_file(project_dir / "file3.md", "# File 3") + + # First sync + report1 = await sync_service.sync(project_dir) + assert len(report1.new) == 3 + + # Delete one file + (project_dir / "file2.md").unlink() + + # Sleep to ensure file operations complete + await sleep_past_watermark() + + # Second sync - should trigger full scan due to decreased file count + report2 = await sync_service.sync(project_dir) + + assert len(report2.deleted) == 1 + assert "file2.md" in report2.deleted + + +@pytest.mark.asyncio +async def test_deletion_triggers_full_scan_multiple_files( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that deleting multiple files triggers full scan.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + await create_test_file(project_dir / "file3.md", "# File 3") + await create_test_file(project_dir / "file4.md", "# File 4") + + # First sync + await sync_service.sync(project_dir) + + # Delete multiple files + (project_dir / "file2.md").unlink() + (project_dir / "file4.md").unlink() + + # Sleep to ensure file operations complete + await sleep_past_watermark() + + # Second sync - should trigger full scan and detect both deletions + report = await sync_service.sync(project_dir) + + assert len(report.deleted) == 2 + assert "file2.md" in report.deleted + assert "file4.md" in report.deleted + + +# ============================================================================== +# Move Detection Tests +# ============================================================================== + + +@pytest.mark.asyncio +async def test_move_detection_requires_full_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that file moves require full scan to be detected (cannot detect in incremental). + + Moves (renames) don't update mtime, so incremental scans can't detect them. + To trigger a full scan for move detection, we need file count to decrease. + This test verifies moves are detected when combined with a deletion. + """ + project_dir = project_config.home + + # Create initial files - include extra file to delete later + old_path = project_dir / "old" / "file.md" + content = dedent( + """ + --- + title: Test File + type: note + --- + # Test File + Distinctive content for move detection + """ + ).strip() + await create_test_file(old_path, content) + await create_test_file(project_dir / "other.md", "# Other\nContent") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure operations complete and watermark is in the past + await sleep_past_watermark() + + # Move file AND delete another to trigger full scan + # Move alone won't work because file count stays same (no full scan) + new_path = project_dir / "new" / "moved.md" + new_path.parent.mkdir(parents=True, exist_ok=True) + old_path.rename(new_path) + (project_dir / "other.md").unlink() # Delete to trigger full scan + + # Second sync - full scan due to deletion, move detected via checksum + report = await sync_service.sync(project_dir) + + assert len(report.moves) == 1 + assert "old/file.md" in report.moves + assert report.moves["old/file.md"] == "new/moved.md" + assert len(report.deleted) == 1 + assert "other.md" in report.deleted + + +@pytest.mark.asyncio +async def test_move_detection_in_full_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that file moves are detected via checksum in full scan.""" + project_dir = project_config.home + + # Create initial files + old_path = project_dir / "old" / "file.md" + content = dedent( + """ + --- + title: Test File + type: note + --- + # Test File + Distinctive content for move detection + """ + ).strip() + await create_test_file(old_path, content) + await create_test_file(project_dir / "other.md", "# Other\nContent") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure operations complete + await sleep_past_watermark() + + # Move file AND delete another to trigger full scan + new_path = project_dir / "new" / "moved.md" + new_path.parent.mkdir(parents=True, exist_ok=True) + old_path.rename(new_path) + (project_dir / "other.md").unlink() + + # Second sync - full scan due to deletion, should still detect move + report = await sync_service.sync(project_dir) + + assert len(report.moves) == 1 + assert "old/file.md" in report.moves + assert report.moves["old/file.md"] == "new/moved.md" + assert len(report.deleted) == 1 + assert "other.md" in report.deleted + + +# ============================================================================== +# Watermark Update Tests +# ============================================================================== + + +@pytest.mark.asyncio +async def test_watermark_updated_after_successful_sync( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that watermark is updated after each successful sync.""" + project_dir = project_config.home + + # Create initial file + await create_test_file(project_dir / "file1.md", "# File 1") + + # Get project before sync + project_before = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + assert project_before.last_scan_timestamp is None + assert project_before.last_file_count is None + + # First sync + sync_start = time.time() + await sync_service.sync(project_dir) + sync_end = time.time() + + # Verify watermark was set + project_after = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + assert project_after.last_scan_timestamp is not None + assert project_after.last_file_count >= 1 # May include config files + + # Watermark should be between sync start and end + assert sync_start <= project_after.last_scan_timestamp <= sync_end + + +@pytest.mark.asyncio +async def test_watermark_uses_sync_start_time( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that watermark uses sync start time, not end time.""" + project_dir = project_config.home + + # Create initial file + await create_test_file(project_dir / "file1.md", "# File 1") + + # First sync - capture timestamps + sync_start = time.time() + await sync_service.sync(project_dir) + sync_end = time.time() + + # Get watermark + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + + # Watermark should be closer to start than end + # (In practice, watermark == sync_start_timestamp captured in sync()) + time_from_start = abs(project.last_scan_timestamp - sync_start) + time_from_end = abs(project.last_scan_timestamp - sync_end) + + assert time_from_start < time_from_end + + +@pytest.mark.asyncio +async def test_watermark_file_count_accurate( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that watermark file count accurately reflects synced files.""" + project_dir = project_config.home + + # Create initial files + await create_test_file(project_dir / "file1.md", "# File 1") + await create_test_file(project_dir / "file2.md", "# File 2") + await create_test_file(project_dir / "file3.md", "# File 3") + + # First sync + await sync_service.sync(project_dir) + + # Verify file count + project1 = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + initial_count = project1.last_file_count + assert initial_count >= 3 # May include config files + + # Add more files + await sleep_past_watermark() + await create_test_file(project_dir / "file4.md", "# File 4") + await create_test_file(project_dir / "file5.md", "# File 5") + + # Second sync + await sync_service.sync(project_dir) + + # Verify updated count increased by 2 + project2 = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + assert project2.last_file_count == initial_count + 2 + + +# ============================================================================== +# Edge Cases and Error Handling +# ============================================================================== + + +@pytest.mark.asyncio +async def test_concurrent_file_changes_handled_gracefully( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that files created/modified during sync are handled correctly. + + Files created during sync (between start and file processing) should be + caught in the next sync, not cause errors in the current sync. + """ + project_dir = project_config.home + + # Create initial file + await create_test_file(project_dir / "file1.md", "# File 1") + + # First sync + await sync_service.sync(project_dir) + + # Sleep to ensure mtime will be newer + await sleep_past_watermark() + + # Create file that will have mtime very close to watermark + # In real scenarios, this could be created during sync + await create_test_file(project_dir / "concurrent.md", "# Concurrent") + + # Should be caught in next sync without errors + report = await sync_service.sync(project_dir) + assert "concurrent.md" in report.new + + +@pytest.mark.asyncio +async def test_empty_directory_handles_incremental_scan( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan handles empty directories correctly.""" + project_dir = project_config.home + + # First sync with empty directory (no user files) + report1 = await sync_service.sync(project_dir) + assert len(report1.new) == 0 + + # Verify watermark was set even for empty directory + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + assert project.last_scan_timestamp is not None + # May have config files, so just check it's set + assert project.last_file_count is not None + + # Second sync - still empty (no new user files) + report2 = await sync_service.sync(project_dir) + assert len(report2.new) == 0 + + +@pytest.mark.asyncio +async def test_incremental_scan_respects_gitignore( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that incremental scan respects .gitignore patterns.""" + project_dir = project_config.home + + # Create .gitignore + (project_dir / ".gitignore").write_text("*.ignored\n.hidden/\n") + + # Reload ignore patterns + from basic_memory.ignore_utils import load_gitignore_patterns + + sync_service._ignore_patterns = load_gitignore_patterns(project_dir) + + # Create files - some should be ignored + await create_test_file(project_dir / "included.md", "# Included") + await create_test_file(project_dir / "excluded.ignored", "# Excluded") + + # First sync + report1 = await sync_service.sync(project_dir) + assert "included.md" in report1.new + assert "excluded.ignored" not in report1.new + + # Sleep and add more files + await sleep_past_watermark() + await create_test_file(project_dir / "included2.md", "# Included 2") + await create_test_file(project_dir / "excluded2.ignored", "# Excluded 2") + + # Second sync - incremental scan should also respect ignore patterns + report2 = await sync_service.sync(project_dir) + assert "included2.md" in report2.new + assert "excluded2.ignored" not in report2.new + + +# ============================================================================== +# Relation Resolution Optimization Tests +# ============================================================================== + + +@pytest.mark.asyncio +async def test_relation_resolution_skipped_when_no_changes( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that relation resolution is skipped when no file changes detected. + + This optimization prevents wasting time resolving relations when there are + no changes, dramatically improving sync performance for large projects. + """ + project_dir = project_config.home + + # Create initial file with wikilink + content = dedent( + """ + --- + title: File with Link + type: note + --- + # File with Link + This links to [[Target File]] + """ + ).strip() + await create_test_file(project_dir / "file1.md", content) + + # First sync - will resolve relations (or leave unresolved) + report1 = await sync_service.sync(project_dir) + assert len(report1.new) == 1 + + # Check that there are unresolved relations (target doesn't exist) + unresolved = await sync_service.relation_repository.find_unresolved_relations() + unresolved_count_before = len(unresolved) + assert unresolved_count_before > 0 # Should have unresolved relation to [[Target File]] + + # Sleep to ensure time passes + await sleep_past_watermark() + + # Second sync - no changes, should skip relation resolution + report2 = await sync_service.sync(project_dir) + assert report2.total == 0 # No changes detected + + # Verify unresolved relations count unchanged (resolution was skipped) + unresolved_after = await sync_service.relation_repository.find_unresolved_relations() + assert len(unresolved_after) == unresolved_count_before + + +@pytest.mark.asyncio +async def test_relation_resolution_runs_when_files_modified( + sync_service: SyncService, project_config: ProjectConfig +): + """Test that relation resolution runs when files are actually modified.""" + project_dir = project_config.home + + # Create file with unresolved wikilink + content1 = dedent( + """ + --- + title: File with Link + type: note + --- + # File with Link + This links to [[Target File]] + """ + ).strip() + await create_test_file(project_dir / "file1.md", content1) + + # First sync + await sync_service.sync(project_dir) + + # Verify unresolved relation exists + unresolved_before = await sync_service.relation_repository.find_unresolved_relations() + assert len(unresolved_before) > 0 + + # Sleep to ensure mtime will be newer + await sleep_past_watermark() + + # Create the target file (should resolve the relation) + content2 = dedent( + """ + --- + title: Target File + type: note + --- + # Target File + This is the target. + """ + ).strip() + await create_test_file(project_dir / "target.md", content2) + + # Second sync - should detect new file and resolve relations + report = await sync_service.sync(project_dir) + assert len(report.new) == 1 + assert "target.md" in report.new + + # Verify relation was resolved (unresolved count decreased) + unresolved_after = await sync_service.relation_repository.find_unresolved_relations() + assert len(unresolved_after) < len(unresolved_before) diff --git a/tests/sync/test_sync_wikilink_issue.py b/tests/sync/test_sync_wikilink_issue.py index 4d90e9a0..2e6241ef 100644 --- a/tests/sync/test_sync_wikilink_issue.py +++ b/tests/sync/test_sync_wikilink_issue.py @@ -13,6 +13,22 @@ async def create_test_file(path: Path, content: str) -> None: path.write_text(content) +async def force_full_scan(sync_service: SyncService) -> None: + """Force next sync to do a full scan by clearing watermark (for testing moves/deletions).""" + if sync_service.entity_repository.project_id is not None: + project = await sync_service.project_repository.find_by_id( + sync_service.entity_repository.project_id + ) + if project: + await sync_service.project_repository.update( + project.id, + { + "last_scan_timestamp": None, + "last_file_count": None, + }, + ) + + @pytest.mark.asyncio async def test_wikilink_modified_status_issue(sync_service: SyncService, project_config): """Test that files with wikilinks don't remain in modified status after sync.""" @@ -52,6 +68,10 @@ This is the target file. target_file_path = project_dir / "another_file.md" await create_test_file(target_file_path, target_content) + # Force full scan to detect the new file + # (file just created may not be newer than watermark due to timing precision) + await force_full_scan(sync_service) + # Sync again after adding target file report3 = await sync_service.sync(project_config.home) assert "another_file.md" in report3.new diff --git a/tests/utils/test_file_utils.py b/tests/utils/test_file_utils.py index c7526a0d..5e7b7b77 100644 --- a/tests/utils/test_file_utils.py +++ b/tests/utils/test_file_utils.py @@ -11,13 +11,11 @@ from basic_memory.file_utils import ( FileWriteError, ParseError, compute_checksum, - ensure_directory, has_frontmatter, parse_frontmatter, remove_frontmatter, sanitize_for_filename, sanitize_for_folder, - update_frontmatter, write_file_atomic, ) @@ -51,15 +49,6 @@ async def test_compute_checksum_error(): await compute_checksum(object()) # pyright: ignore [reportArgumentType] -@pytest.mark.asyncio -async def test_ensure_directory(tmp_path: Path): - """Test directory creation.""" - test_dir = tmp_path / "test_dir" - await ensure_directory(test_dir) - assert test_dir.exists() - assert test_dir.is_dir() - - @pytest.mark.asyncio async def test_write_file_atomic(tmp_path: Path): """Test atomic file writing.""" @@ -188,108 +177,6 @@ title: Test""") assert "Invalid frontmatter format" in str(exc.value) -@pytest.mark.asyncio -async def test_update_frontmatter(tmp_path: Path): - """Test updating frontmatter in a file.""" - test_file = tmp_path / "test.md" - - # Test 1: Add frontmatter to file without any - content = "# Test Content\n\nSome content here" - test_file.write_text(content) - - updates = {"title": "Test", "type": "note"} - checksum = await update_frontmatter(test_file, updates) - - # Verify content - updated = test_file.read_text(encoding="utf-8") - assert "title: Test" in updated - assert "type: note" in updated - assert "Test Content" in updated - assert "Some content here" in updated - - # Verify structure - fm = parse_frontmatter(updated) - assert fm == updates - assert remove_frontmatter(updated).strip() == content - - # Test 2: Update existing frontmatter - updates = {"type": "doc", "tags": ["test"]} - new_checksum = await update_frontmatter(test_file, updates) - - # Verify checksum changed - assert new_checksum != checksum - - # Verify content - updated = test_file.read_text(encoding="utf-8") - fm = parse_frontmatter(updated) - assert fm == {"title": "Test", "type": "doc", "tags": ["test"]} - assert "Test Content" in updated - - # Test 3: Update with empty dict shouldn't change anything - checksum_before = await compute_checksum(test_file.read_text(encoding="utf-8")) - new_checksum = await update_frontmatter(test_file, {}) - assert new_checksum == checksum_before - - # Test 4: Handle multi-line content properly - content = """# Heading - -Some content - -## Section -- Point 1 -- Point 2 - -### Subsection -More content here""" - - test_file.write_text(content) - await update_frontmatter(test_file, {"title": "Test"}) - - updated = test_file.read_text(encoding="utf-8") - assert remove_frontmatter(updated).strip() == content - - -@pytest.mark.asyncio -async def test_update_frontmatter_errors(tmp_path: Path): - """Test error handling in update_frontmatter.""" - - # Test 1: Invalid file path - nonexistent = tmp_path / "nonexistent" / "test.md" - with pytest.raises(FileError): - await update_frontmatter(nonexistent, {"title": "Test"}) - - -@pytest.mark.asyncio -async def test_update_frontmatter_handles_malformed_yaml(tmp_path: Path): - """Test that update_frontmatter handles malformed YAML gracefully (issue #378).""" - test_file = tmp_path / "test.md" - - # Create file with malformed YAML frontmatter (colon in title breaks YAML) - content = """--- -title: KB: Something ---- - -# Test Content - -Some content here""" - test_file.write_text(content) - - # Should handle gracefully and treat as having no frontmatter - updates = {"title": "Fixed Title", "type": "note"} - await update_frontmatter(test_file, updates) - - # Verify file was updated successfully - updated = test_file.read_text(encoding="utf-8") - assert "title: Fixed Title" in updated - assert "type: note" in updated - assert "Test Content" in updated - assert "Some content here" in updated - - # Verify new frontmatter is valid - fm = parse_frontmatter(updated) - assert fm == updates - - @pytest.mark.asyncio def test_sanitize_for_filename_removes_invalid_characters(): # Test all invalid characters listed in the regex diff --git a/tests/utils/test_utf8_handling.py b/tests/utils/test_utf8_handling.py index 6c08e4c0..16450eb9 100644 --- a/tests/utils/test_utf8_handling.py +++ b/tests/utils/test_utf8_handling.py @@ -2,7 +2,6 @@ import pytest -from basic_memory import file_utils from basic_memory.file_utils import compute_checksum, write_file_atomic @@ -56,8 +55,11 @@ async def test_write_utf8_characters(tmp_path): @pytest.mark.asyncio -async def test_frontmatter_with_utf8(tmp_path): +async def test_frontmatter_with_utf8(tmp_path, sync_service): """Test handling of frontmatter with UTF-8 characters.""" + # Use FileService from sync_service + file_service = sync_service.file_service + # Create a test file with frontmatter containing UTF-8 test_path = tmp_path / "frontmatter_utf8.md" @@ -65,7 +67,7 @@ async def test_frontmatter_with_utf8(tmp_path): content = """--- title: UTF-8 测试文件 (Test File) author: José García -keywords: +keywords: - тестирование - 테스트 - 测试 @@ -79,8 +81,8 @@ This file has UTF-8 characters in the frontmatter. # Write the file await write_file_atomic(test_path, content) - # Update frontmatter with more UTF-8 - await file_utils.update_frontmatter( + # Update frontmatter with more UTF-8 using FileService + await file_service.update_frontmatter( test_path, { "description": "这是一个测试文件 (This is a test file)", diff --git a/uv.lock b/uv.lock index edd64d7b..5ac9e5e9 100644 --- a/uv.lock +++ b/uv.lock @@ -102,6 +102,7 @@ dependencies = [ { name = "fastmcp" }, { name = "greenlet" }, { name = "icecream" }, + { name = "logfire" }, { name = "loguru" }, { name = "markdown-it-py" }, { name = "mcp" }, @@ -145,6 +146,7 @@ requires-dist = [ { name = "fastmcp", specifier = ">=2.10.2" }, { name = "greenlet", specifier = ">=3.1.1" }, { name = "icecream", specifier = ">=2.1.3" }, + { name = "logfire", specifier = ">=0.73.0" }, { name = "loguru", specifier = ">=0.7.3" }, { name = "markdown-it-py", specifier = ">=3.0.0" }, { name = "mcp", specifier = ">=1.2.0" }, @@ -609,6 +611,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/16/b71171e97ec7b4ded8669542f4369d88d5a289e2704efbbde51e858e062a/gevent-25.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0bacf89a65489d26c7087669af89938d5bfd9f7afb12a07b57855b9fad6ccbd0", size = 2937113, upload-time = "2025-05-12T11:12:03.191Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, +] + [[package]] name = "greenlet" version = "3.2.4" @@ -734,6 +748,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + [[package]] name = "iniconfig" version = "2.1.0" @@ -821,6 +847,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/1e/fb441c07b6662ec1fc92b249225ba6e6e5221b05623cb0131d082f782edc/lazy_object_proxy-1.11.0-py3-none-any.whl", hash = "sha256:a56a5093d433341ff7da0e89f9b486031ccd222ec8e52ec84d0ec1cdc819674b", size = 16635, upload-time = "2025-04-16T16:53:47.198Z" }, ] +[[package]] +name = "logfire" +version = "4.13.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "executing" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-sdk" }, + { name = "protobuf" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/51/004bbe0276fcecfea8d4a4c76ad33426c9b34428e0723f22c6bb801dbc22/logfire-4.13.2.tar.gz", hash = "sha256:4e756e140c3b8fd25653d20437ebcb75734975f5382de6ae28be775c75575d95", size = 547796, upload-time = "2025-10-13T16:17:53.392Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/5f/0803848cc5ce524ff830e5f2a2f1400fd6ee72be705d87d0432cec42b1e4/logfire-4.13.2-py3-none-any.whl", hash = "sha256:887e99897a1818864aa5bfc595b02c93264ce23d1860866369eff6b6e2dde1c6", size = 228152, upload-time = "2025-10-13T16:17:50.641Z" }, +] + [[package]] name = "loguru" version = "0.7.3" @@ -1006,6 +1050,103 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/dd/b3fd642260cb17532f66cc1e8250f3507d1e580483e209dc1e9d13bd980d/openapi_spec_validator-0.7.2-py3-none-any.whl", hash = "sha256:4bbdc0894ec85f1d1bea1d6d9c8b2c3c8d7ccaa13577ef40da9c006c9fd0eb60", size = 39713, upload-time = "2025-06-07T14:48:54.077Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/04/05040d7ce33a907a2a02257e601992f0cdf11c73b33f13c4492bf6c3d6d5/opentelemetry_api-1.37.0.tar.gz", hash = "sha256:540735b120355bd5112738ea53621f8d5edb35ebcd6fe21ada3ab1c61d1cd9a7", size = 64923, upload-time = "2025-09-11T10:29:01.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6c/10018cbcc1e6fff23aac67d7fd977c3d692dbe5f9ef9bb4db5c1268726cc/opentelemetry_exporter_otlp_proto_common-1.37.0.tar.gz", hash = "sha256:c87a1bdd9f41fdc408d9cc9367bb53f8d2602829659f2b90be9f9d79d0bfe62c", size = 20430, upload-time = "2025-09-11T10:29:03.605Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/13/b4ef09837409a777f3c0af2a5b4ba9b7af34872bc43609dda0c209e4060d/opentelemetry_exporter_otlp_proto_common-1.37.0-py3-none-any.whl", hash = "sha256:53038428449c559b0c564b8d718df3314da387109c4d36bd1b94c9a641b0292e", size = 18359, upload-time = "2025-09-11T10:28:44.939Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/e3/6e320aeb24f951449e73867e53c55542bebbaf24faeee7623ef677d66736/opentelemetry_exporter_otlp_proto_http-1.37.0.tar.gz", hash = "sha256:e52e8600f1720d6de298419a802108a8f5afa63c96809ff83becb03f874e44ac", size = 17281, upload-time = "2025-09-11T10:29:04.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/e9/70d74a664d83976556cec395d6bfedd9b85ec1498b778367d5f93e373397/opentelemetry_exporter_otlp_proto_http-1.37.0-py3-none-any.whl", hash = "sha256:54c42b39945a6cc9d9a2a33decb876eabb9547e0dcb49df090122773447f1aef", size = 19576, upload-time = "2025-09-11T10:28:46.726Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.58b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/36/7c307d9be8ce4ee7beb86d7f1d31027f2a6a89228240405a858d6e4d64f9/opentelemetry_instrumentation-0.58b0.tar.gz", hash = "sha256:df640f3ac715a3e05af145c18f527f4422c6ab6c467e40bd24d2ad75a00cb705", size = 31549, upload-time = "2025-09-11T11:42:14.084Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/db/5ff1cd6c5ca1d12ecf1b73be16fbb2a8af2114ee46d4b0e6d4b23f4f4db7/opentelemetry_instrumentation-0.58b0-py3-none-any.whl", hash = "sha256:50f97ac03100676c9f7fc28197f8240c7290ca1baa12da8bfbb9a1de4f34cc45", size = 33019, upload-time = "2025-09-11T11:41:00.624Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/ea/a75f36b463a36f3c5a10c0b5292c58b31dbdde74f6f905d3d0ab2313987b/opentelemetry_proto-1.37.0.tar.gz", hash = "sha256:30f5c494faf66f77faeaefa35ed4443c5edb3b0aa46dad073ed7210e1a789538", size = 46151, upload-time = "2025-09-11T10:29:11.04Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/25/f89ea66c59bd7687e218361826c969443c4fa15dfe89733f3bf1e2a9e971/opentelemetry_proto-1.37.0-py3-none-any.whl", hash = "sha256:8ed8c066ae8828bbf0c39229979bdf583a126981142378a9cbe9d6fd5701c6e2", size = 72534, upload-time = "2025-09-11T10:28:56.831Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/62/2e0ca80d7fe94f0b193135375da92c640d15fe81f636658d2acf373086bc/opentelemetry_sdk-1.37.0.tar.gz", hash = "sha256:cc8e089c10953ded765b5ab5669b198bbe0af1b3f89f1007d19acd32dc46dda5", size = 170404, upload-time = "2025-09-11T10:29:11.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/62/9f4ad6a54126fb00f7ed4bb5034964c6e4f00fcd5a905e115bd22707e20d/opentelemetry_sdk-1.37.0-py3-none-any.whl", hash = "sha256:8f3c3c22063e52475c5dbced7209495c2c16723d016d39287dfc215d1771257c", size = 131941, upload-time = "2025-09-11T10:28:57.83Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.58b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/1b/90701d91e6300d9f2fb352153fb1721ed99ed1f6ea14fa992c756016e63a/opentelemetry_semantic_conventions-0.58b0.tar.gz", hash = "sha256:6bd46f51264279c433755767bb44ad00f1c9e2367e1b42af563372c5a6fa0c25", size = 129867, upload-time = "2025-09-11T10:29:12.597Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -1108,6 +1249,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/ff/64a6c8f420818bb873713988ca5492cba3a7946be57e027ac63495157d97/protobuf-6.33.0.tar.gz", hash = "sha256:140303d5c8d2037730c548f8c7b93b20bb1dc301be280c378b82b8894589c954", size = 443463, upload-time = "2025-10-15T20:39:52.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/ee/52b3fa8feb6db4a833dfea4943e175ce645144532e8a90f72571ad85df4e/protobuf-6.33.0-cp310-abi3-win32.whl", hash = "sha256:d6101ded078042a8f17959eccd9236fb7a9ca20d3b0098bbcb91533a5680d035", size = 425593, upload-time = "2025-10-15T20:39:40.29Z" }, + { url = "https://files.pythonhosted.org/packages/7b/c6/7a465f1825872c55e0341ff4a80198743f73b69ce5d43ab18043699d1d81/protobuf-6.33.0-cp310-abi3-win_amd64.whl", hash = "sha256:9a031d10f703f03768f2743a1c403af050b6ae1f3480e9c140f39c45f81b13ee", size = 436882, upload-time = "2025-10-15T20:39:42.841Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a9/b6eee662a6951b9c3640e8e452ab3e09f117d99fc10baa32d1581a0d4099/protobuf-6.33.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:905b07a65f1a4b72412314082c7dbfae91a9e8b68a0cc1577515f8df58ecf455", size = 427521, upload-time = "2025-10-15T20:39:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/10/35/16d31e0f92c6d2f0e77c2a3ba93185130ea13053dd16200a57434c882f2b/protobuf-6.33.0-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e0697ece353e6239b90ee43a9231318302ad8353c70e6e45499fa52396debf90", size = 324445, upload-time = "2025-10-15T20:39:44.932Z" }, + { url = "https://files.pythonhosted.org/packages/e6/eb/2a981a13e35cda8b75b5585aaffae2eb904f8f351bdd3870769692acbd8a/protobuf-6.33.0-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:e0a1715e4f27355afd9570f3ea369735afc853a6c3951a6afe1f80d8569ad298", size = 339159, upload-time = "2025-10-15T20:39:46.186Z" }, + { url = "https://files.pythonhosted.org/packages/21/51/0b1cbad62074439b867b4e04cc09b93f6699d78fd191bed2bbb44562e077/protobuf-6.33.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:35be49fd3f4fefa4e6e2aacc35e8b837d6703c37a2168a55ac21e9b1bc7559ef", size = 323172, upload-time = "2025-10-15T20:39:47.465Z" }, + { url = "https://files.pythonhosted.org/packages/07/d1/0a28c21707807c6aacd5dc9c3704b2aa1effbf37adebd8caeaf68b17a636/protobuf-6.33.0-py3-none-any.whl", hash = "sha256:25c9e1963c6734448ea2d308cfa610e692b801304ba0908d7bfa564ac5132995", size = 170477, upload-time = "2025-10-15T20:39:51.311Z" }, +] + [[package]] name = "pybars3" version = "0.9.7" @@ -2045,6 +2201,64 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, ] +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" }, + { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" }, + { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" }, + { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" }, + { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, + { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, + { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, + { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" }, + { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] + [[package]] name = "zope-event" version = "5.1.1"