status command

src/basic_memory/cli/commands/status.py

@@ -4,6 +4,7 @@ import asyncio
 from typing import Set, Dict
 
 import typer
+from loguru import logger
 from rich.console import Console
 from rich.panel import Panel
 from rich.tree import Tree
@@ -179,5 +180,6 @@ def status(
         sync_service = asyncio.run(get_sync_service())
         asyncio.run(run_status(sync_service, verbose))
     except Exception as e:
+        logger.error(f"Error checking status: {e}")
         typer.echo(f"Error checking status: {e}", err=True)
         raise typer.Exit(1)

src/basic_memory/services/file_sync_service.py

@@ -2,7 +2,7 @@
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Set, Dict, Protocol, TypeVar, Generic, List, Optional
+from typing import Set, Dict, Protocol, TypeVar, Generic, Optional
 
 from loguru import logger
 
@@ -16,7 +16,6 @@ class FileState:
     """State of a file including path and checksum info."""
     path: str
     checksum: str
-    normalized_path: str  # For comparison
     moved_from: Optional[str] = None
 
 
@@ -37,7 +36,9 @@ class SyncReport:
 class DbRecord(Protocol):
     """Protocol for database records with path and checksum."""
     @property
-    def path(self) -> str: ...
+    def file_path(self) -> Optional[str]: ...
+    @property
+    def path_id(self) -> str: ...
     @property
     def checksum(self) -> str: ...
 
@@ -94,9 +95,7 @@ class FileSyncService:
     async def find_changes(
             self,
             directory: Path,
-            get_records: callable,
-            normalize_path: callable = lambda x: x,
-            get_record_path: callable = lambda x: x.path_id
+            get_records: callable
     ) -> SyncReport:
         """
         Find changes between filesystem and database.
@@ -104,8 +103,6 @@ class FileSyncService:
         Args:
             directory: Directory to check
             get_records: Function to get database records
-            normalize_path: Function to normalize paths for comparison
-            get_record_path: Function to get path from record
 
         Returns:
             SyncReport detailing changes
@@ -121,34 +118,32 @@ class FileSyncService:
         for path, checksum in current_files.items():
             report.checksums[path] = checksum
 
-        # Build DB state with normalized paths
+        # Build DB state - use path_id if file_path is NULL
         db_records = await get_records()
-        db_files: Dict[str, tuple[str, str]] = {
-            normalize_path(get_record_path(record)): (get_record_path(record), record.checksum)
-            for record in db_records
-        }
-        
-        logger.debug("Files from database:")
-        for norm_path, (orig_path, checksum) in sorted(db_files.items()):
-            logger.debug(f"  {norm_path} ({checksum[:8]}) [original: {orig_path}]")
+        db_files = {}
+        for record in db_records:
+            path = record.file_path if record.file_path is not None else record.path_id
+            db_files[path] = (path, record.checksum)
 
-        # Track files by checksum to detect moves
-        checksum_locations: Dict[str, List[str]] = {}
-        for path, checksum in current_files.items():
-            norm_path = normalize_path(path)
-            locations = checksum_locations.setdefault(checksum, [])
-            locations.append(norm_path)
+        logger.debug("Files from database:")
+        for path, (_, checksum) in sorted(db_files.items()):
+            logger.debug(f"  {path} ({checksum[:8]})")
+
+        # Track files by checksum for move detection
+        db_paths_by_checksum = {}
+        for path, (_, checksum) in db_files.items():
+            paths = db_paths_by_checksum.setdefault(checksum, [])
+            paths.append(path)
 
         processed_current = set()
         processed_db = set()
 
         # First pass - check for unchanged and modified files
         for curr_path, curr_checksum in current_files.items():
-            norm_curr_path = normalize_path(curr_path)
-            if norm_curr_path in db_files:
-                db_orig_path, db_checksum = db_files[norm_curr_path]
-                processed_current.add(norm_curr_path)
-                processed_db.add(norm_curr_path)
+            if curr_path in db_files:
+                _, db_checksum = db_files[curr_path]
+                processed_current.add(curr_path)
+                processed_db.add(curr_path)
                 
                 if curr_checksum != db_checksum:
                     logger.debug(f"Modified: {curr_path} (checksum changed)")
@@ -156,38 +151,35 @@ class FileSyncService:
 
         # Second pass - look for moves
         for curr_path, curr_checksum in current_files.items():
-            norm_curr_path = normalize_path(curr_path)
-            if norm_curr_path in processed_current:
+            if curr_path in processed_current:
                 continue
 
-            # Look for files with same checksum in DB
+            # Look for any files with same checksum in DB
             was_move = False
-            for db_norm_path, (db_orig_path, db_checksum) in db_files.items():
-                if db_norm_path in processed_db:
-                    continue
-                if curr_checksum == db_checksum:
-                    logger.debug(f"Moved: {db_orig_path} -> {curr_path}")
-                    report.moved[curr_path] = FileState(
-                        path=curr_path,
-                        checksum=curr_checksum,
-                        normalized_path=norm_curr_path,
-                        moved_from=db_orig_path
-                    )
-                    processed_current.add(norm_curr_path)
-                    processed_db.add(db_norm_path)
-                    was_move = True
-                    break
-            
+            if curr_checksum in db_paths_by_checksum:
+                for db_path in db_paths_by_checksum[curr_checksum]:
+                    if db_path not in processed_db:
+                        logger.debug(f"Moved: {db_path} -> {curr_path}")
+                        report.moved[curr_path] = FileState(
+                            path=curr_path,
+                            checksum=curr_checksum,
+                            moved_from=db_path
+                        )
+                        processed_current.add(curr_path)
+                        processed_db.add(db_path)
+                        was_move = True
+                        break
+
             if not was_move:
                 logger.debug(f"New: {curr_path}")
                 report.new.add(curr_path)
-                processed_current.add(norm_curr_path)
+                processed_current.add(curr_path)
 
         # Remaining DB files must be deleted
-        for db_norm_path, (db_orig_path, _) in db_files.items():
-            if db_norm_path not in processed_db:
-                logger.debug(f"Deleted: {db_orig_path}")
-                report.deleted.add(db_orig_path)
+        for path, (db_path, _) in db_files.items():
+            if path not in processed_db:
+                logger.debug(f"Deleted: {db_path}")
+                report.deleted.add(db_path)
 
         # Log summary
         logger.debug(f"Changes found: {report.total_changes}")
@@ -200,35 +192,14 @@ class FileSyncService:
 
     async def find_document_changes(self, directory: Path) -> SyncReport:
         """Find changes in document directory."""
-        def normalize_doc_path(path: str) -> str:
-            """Normalize document paths."""
-            return str(Path(path))
-
         return await self.find_changes(
             directory=directory,
-            get_records=self.document_repository.find_all,
-            normalize_path=normalize_doc_path
+            get_records=self.document_repository.find_all
         )
 
     async def find_knowledge_changes(self, directory: Path) -> SyncReport:
         """Find changes in knowledge directory."""
-        def normalize_entity_path(path: str) -> str:
-            """Normalize entity paths."""
-            path = path.lower()
-            if path.endswith('.md'):
-                path = path[:-3]
-            return path
-
-        def get_entity_path(entity) -> str:
-            """Get path from entity record."""
-            path = entity.path_id
-            if not path.endswith('.md'):
-                path = f"{path}.md"
-            return path
-
         return await self.find_changes(
             directory=directory,
-            get_records=self.entity_repository.find_all,
-            normalize_path=normalize_entity_path,
-            get_record_path=get_entity_path
+            get_records=self.entity_repository.find_all
         )

tests/services/test_file_sync_service.py

@@ -0,0 +1,200 @@
+"""Tests for file sync service."""
+
+import asyncio
+from pathlib import Path
+from typing import AsyncGenerator
+import pytest
+import pytest_asyncio
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from basic_memory import db
+from basic_memory.repository import DocumentRepository, EntityRepository
+from basic_memory.services import FileSyncService
+from basic_memory.utils.file_utils import compute_checksum
+
+@pytest_asyncio.fixture
+def temp_dir(test_config):
+    return test_config.home
+
+
+@pytest.mark.asyncio
+async def create_test_file(path: Path, content: str = "test content") -> None:
+    """Create a test file with given content."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+
+@pytest.mark.asyncio
+async def test_scan_directory_empty(
+    file_sync_service: FileSyncService,
+    temp_dir: Path
+):
+    """Test scanning empty directory."""
+    files = await file_sync_service.scan_directory(temp_dir)
+    assert len(files) == 0
+
+@pytest.mark.asyncio
+async def test_scan_directory_with_files(
+    file_sync_service: FileSyncService,
+    temp_dir: Path
+):
+    """Test scanning directory with markdown files."""
+    # Create test files
+    await create_test_file(temp_dir / "test1.md", "content 1")
+    await create_test_file(temp_dir / "test2.md", "content 2")
+    await create_test_file(temp_dir / "not-markdown.txt", "ignore me")
+
+    files = await file_sync_service.scan_directory(temp_dir)
+    assert len(files) == 2
+    assert "test1.md" in files
+    assert "test2.md" in files
+    assert "not-markdown.txt" not in files
+
+@pytest.mark.asyncio
+async def test_find_new_files(
+    file_sync_service: FileSyncService,
+    temp_dir: Path
+):
+    """Test detection of new files."""
+    # Create new file
+    await create_test_file(temp_dir / "new_file.md", "new content")
+
+    changes = await file_sync_service.find_changes(
+        directory=temp_dir,
+        get_records=lambda: asyncio.sleep(0, [])  # Empty DB
+    )
+    
+    assert len(changes.new) == 1
+    assert "new_file.md" in changes.new
+    assert len(changes.modified) == 0
+    assert len(changes.deleted) == 0
+    assert len(changes.moved) == 0
+
+@pytest.mark.asyncio
+async def test_find_case_sensitive_move(
+    file_sync_service: FileSyncService,
+    temp_dir: Path,
+    session_maker: AsyncGenerator[AsyncSession, None]
+):
+    """Test detection of case-sensitive file moves."""
+    # Create and track original file
+    original_path = "Original.md"
+    await create_test_file(temp_dir / original_path, "test content")
+    original_checksum = await compute_checksum("test content")
+    
+    # Add to DB with original path
+    async with db.scoped_session(session_maker) as session:
+        await session.execute(text(
+            f"INSERT INTO document (path_id, file_path, checksum) VALUES ('{original_path.lower()}', '{original_path}', '{original_checksum}')"))
+        await session.commit()
+
+        # Rename file (case change only)
+        new_path = "ORIGINAL.md"
+        (temp_dir / original_path).rename(temp_dir / new_path)
+    
+        async def get_records():
+            return session.execute(
+                text("SELECT * FROM document")
+            ).fetchall()
+
+        changes = await file_sync_service.find_changes(
+            directory=temp_dir,
+            get_records=(await get_records),
+            normalize_path=lambda p: p.lower()
+        )
+
+    assert len(changes.moved) == 1
+    assert new_path in changes.moved
+    assert changes.moved[new_path].moved_from == original_path
+    assert len(changes.new) == 0
+    assert len(changes.modified) == 0
+    assert len(changes.deleted) == 0
+
+
+
+
+@pytest.mark.asyncio
+async def test_find_modified_files(
+    file_sync_service: FileSyncService,
+    temp_dir: Path,
+    session_maker: AsyncGenerator[AsyncSession, None]
+):
+    """Test detection of modified files."""
+    # Create and track original file
+    path = "test.md"
+    await create_test_file(temp_dir / path, "original content")
+    original_checksum = await compute_checksum("original content")
+    
+    # Add to DB
+    async with db.scoped_session(session_maker) as session:
+        await session.execute(
+            "INSERT INTO document (path_id, file_path, checksum) VALUES (?, ?, ?)",
+            [path.lower(), path, original_checksum]
+        )
+        await session.commit()
+
+        # Modify file
+        await create_test_file(temp_dir / path, "modified content")
+    
+        changes = await file_sync_service.find_changes(
+            directory=temp_dir,
+            get_records=lambda: session.execute(
+                "SELECT * FROM document"
+            ).fetchall(),
+            normalize_path=lambda p: p.lower()
+        )
+
+    assert len(changes.modified) == 1
+    assert path in changes.modified
+    assert len(changes.new) == 0
+    assert len(changes.deleted) == 0
+    assert len(changes.moved) == 0
+
+@pytest.mark.asyncio
+async def test_find_deleted_files(
+    file_sync_service: FileSyncService,
+    temp_dir: Path,
+    session_maker: AsyncGenerator[AsyncSession, None]
+):
+    """Test detection of deleted files."""
+    # Add file to DB that doesn't exist
+    missing_path = "deleted.md"
+    async with db.scoped_session(session_maker) as session:
+        await session.execute(
+            "INSERT INTO document (path_id, file_path, checksum) VALUES (?, ?, ?)",
+            [missing_path.lower(), missing_path, "any-checksum"]
+        )
+        await session.commit()
+
+        changes = await file_sync_service.find_changes(
+            directory=temp_dir,
+            get_records=lambda: session.execute(
+                "SELECT * FROM document"
+            ).fetchall(),
+            normalize_path=lambda p: p.lower()
+        )
+
+    assert len(changes.deleted) == 1
+    assert missing_path in changes.deleted
+    assert len(changes.new) == 0
+    assert len(changes.modified) == 0
+    assert len(changes.moved) == 0
+
+@pytest.mark.asyncio
+async def test_path_normalization(
+    file_sync_service: FileSyncService,
+    temp_dir: Path
+):
+    """Test path normalization for different formats."""
+    paths = [
+        ("my file.md", "my_file"),          # Spaces
+        ("MyFile.md", "myfile"),            # Case
+        ("my/file.md", "my/file"),          # Subdirectories
+        ("MY/FILE.md", "my/file"),          # Mixed case dirs
+        ("my//file.md", "my/file"),         # Extra slashes
+        ("./my/file.md", "my/file"),        # Current dir
+    ]
+
+    for file_path, expected_norm in paths:
+        norm_path = file_sync_service.normalize_path(file_path)
+        assert norm_path == expected_norm, f"Failed to normalize {file_path}"