diff --git a/src/basic_memory/file_utils.py b/src/basic_memory/file_utils.py index a08ac82b..e14cc7c5 100644 --- a/src/basic_memory/file_utils.py +++ b/src/basic_memory/file_utils.py @@ -2,7 +2,7 @@ import hashlib from pathlib import Path -from typing import Dict, Any +from typing import Dict, Any, Union import yaml from loguru import logger @@ -26,12 +26,12 @@ class ParseError(FileError): pass -async def compute_checksum(content: str) -> str: +async def compute_checksum(content: Union[str, bytes]) -> str: """ Compute SHA-256 checksum of content. Args: - content: Text content to hash + content: Content to hash (either text string or bytes) Returns: SHA-256 hex digest @@ -40,12 +40,13 @@ async def compute_checksum(content: str) -> str: FileError: If checksum computation fails """ try: - return hashlib.sha256(content.encode()).hexdigest() + if isinstance(content, str): + content = content.encode() + return hashlib.sha256(content).hexdigest() except Exception as e: # pragma: no cover logger.error(f"Failed to compute checksum: {e}") raise FileError(f"Failed to compute checksum: {e}") - async def ensure_directory(path: Path) -> None: """ Ensure directory exists, creating if necessary. diff --git a/src/basic_memory/services/file_service.py b/src/basic_memory/services/file_service.py index 1928b631..275d11cb 100644 --- a/src/basic_memory/services/file_service.py +++ b/src/basic_memory/services/file_service.py @@ -1,4 +1,5 @@ """Service for file operations with checksum tracking.""" + import mimetypes from os import stat_result from pathlib import Path @@ -7,6 +8,7 @@ from typing import Tuple, Union, Dict, Any from loguru import logger from basic_memory import file_utils +from basic_memory.file_utils import FileError from basic_memory.markdown.markdown_processor import MarkdownProcessor from basic_memory.models import Entity as EntityModel from basic_memory.schemas import Entity as EntitySchema @@ -185,33 +187,39 @@ class FileService: full_path = path if path.is_absolute() else self.base_path / path return await file_utils.update_frontmatter(full_path, updates) - async def compute_checksum(self, path: Union[Path, str]) -> str: - """ - Compute SHA-256 checksum of content. - """ + async def compute_checksum(self, path: Union[str, Path]) -> str: + """Compute checksum for a file.""" path = Path(path) full_path = path if path.is_absolute() else self.base_path / path - - # TODO checksum for binary files - return await file_utils.compute_checksum(full_path.read_text()) - + try: + if self.is_markdown(path): + # read str + content = await self.read_file(full_path) + else: + # read bytes + content = full_path.read_bytes() + return await file_utils.compute_checksum(content) + + except Exception as e: + logger.error(f"Failed to compute checksum for {path}: {e}") + raise FileError(f"Failed to compute checksum for {path}: {e}") def file_stats(self, path: Union[Path, str]) -> stat_result: """ Return file stats for a given path. - :param path: - :return: + :param path: + :return: """ path = Path(path) full_path = path if path.is_absolute() else self.base_path / path # get file timestamps return full_path.stat() - async def content_type(self, path: Union[Path, str]) -> stat_result: + def content_type(self, path: Union[Path, str]) -> str: """ Return content_type for a given path. - :param path: - :return: + :param path: + :return: """ path = Path(path) full_path = path if path.is_absolute() else self.base_path / path @@ -220,10 +228,10 @@ class FileService: content_type = mime_type or "text/plain" return content_type - async def is_markdown(self, path: Union[Path, str]) -> stat_result: + def is_markdown(self, path: Union[Path, str]) -> stat_result: """ Return content_type for a given path. - :param path: - :return: + :param path: + :return: """ return self.content_type(path) == "text/markdown" diff --git a/src/basic_memory/sync/sync_service.py b/src/basic_memory/sync/sync_service.py index c20d3a98..6d863d2f 100644 --- a/src/basic_memory/sync/sync_service.py +++ b/src/basic_memory/sync/sync_service.py @@ -3,6 +3,7 @@ import mimetypes from dataclasses import dataclass from dataclasses import field +from datetime import datetime from pathlib import Path from typing import Set, Dict, Sequence from typing import Tuple @@ -31,16 +32,15 @@ class SyncReport: checksums: Current checksums for files on disk """ - total: int = 0 # We keep paths as strings in sets/dicts for easier serialization new: Set[str] = field(default_factory=set) modified: Set[str] = field(default_factory=set) deleted: Set[str] = field(default_factory=set) moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum - + @property - def total_changes(self) -> int: + def total(self) -> int: """Total number of changes.""" return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves) @@ -59,7 +59,6 @@ class ScanResult: errors: Dict[str, str] = field(default_factory=dict) - class SyncService: """Syncs documents and knowledge files with database.""" @@ -88,10 +87,7 @@ class SyncService: :param db_records: the data from the db """ db_records = await self.entity_repository.find_all() - return { - r.file_path: r.checksum or "" - for r in db_records - } + return {r.file_path: r.checksum or "" for r in db_records} async def sync(self, directory: Path) -> SyncReport: """Sync all files with database.""" @@ -125,7 +121,6 @@ class SyncService: # if checksums don't match for the same path, its modified if local_checksum_for_db_path and db_checksum != local_checksum_for_db_path: report.modified.add(db_path) - # check if it's moved or deleted if not local_checksum_for_db_path: @@ -141,11 +136,11 @@ class SyncService: report.deleted.add(db_path) # order of sync matters to resolve relations effectively - + # sync moves first for old_path, new_path in report.moves.items(): await self.handle_move(old_path, new_path) - + # deleted next for path in report.deleted: await self.handle_delete(path) @@ -196,26 +191,21 @@ class SyncService: if new: # Create entity with final permalink logger.debug(f"Creating new entity from markdown: {path}") - await self.entity_service.create_entity_from_markdown( - Path(path), entity_markdown - ) - + await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown) # otherwise we need to update the entity and observations else: logger.debug(f"Updating entity from markdown: {path}") - await self.entity_service.update_entity_and_observations( - Path(path), entity_markdown - ) - + await self.entity_service.update_entity_and_observations(Path(path), entity_markdown) + # Update relations and search index entity = await self.entity_service.update_entity_relations(path, entity_markdown) - + # set checksum await self.entity_repository.update(entity.id, {"checksum": checksum}) return entity, checksum - async def sync_regular_file(self, path: Path, new: bool = True) -> Tuple[Entity, str]: + async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Entity, str]: """Sync a non-markdown file with basic tracking.""" checksum = await self.file_service.compute_checksum(path) @@ -225,20 +215,22 @@ class SyncService: # get file timestamps file_stats = self.file_service.file_stats(path) + created=datetime.fromtimestamp(file_stats.st_ctime) + modified=datetime.fromtimestamp(file_stats.st_mtime) # get mime type - mime_type, _ = mimetypes.guess_type(path.name) - content_type = mime_type or "text/plain" + content_type = self.file_service.content_type(path) + file_path = Path(path) entity = await self.entity_repository.add( Entity( entity_type="file", file_path=path, permalink=permalink, checksum=checksum, - title=path.name, - created_at=file_stats.st_ctime, - updated_at=file_stats.st_mtime, + title=file_path.name, + created_at=created, + updated_at=modified, content_type=content_type, ) ) @@ -248,7 +240,6 @@ class SyncService: entity.id, {"file_path": path, "checksum": checksum} ) - await self.search_service.index_entity(entity) return entity, checksum async def handle_delete(self, file_path: str): @@ -272,14 +263,12 @@ class SyncService: for permalink in permalinks: await self.search_service.delete_by_permalink(permalink) - async def handle_move(self, old_path, new_path ): + async def handle_move(self, old_path, new_path): logger.debug(f"Moving entity: {old_path} -> {new_path}") entity = await self.entity_repository.get_by_file_path(old_path) if entity: # Update file_path but keep the same permalink for link stability - updated = await self.entity_repository.update( - entity.id, {"file_path": new_path} - ) + updated = await self.entity_repository.update(entity.id, {"file_path": new_path}) # update search index await self.search_service.index_entity(updated) @@ -312,7 +301,6 @@ class SyncService: async def scan_directory(self, directory: Path) -> ScanResult: """ Scan directory for markdown files and their checksums. - Only processes .md files, logs and skips others. Args: directory: Directory to scan @@ -327,16 +315,18 @@ class SyncService: logger.debug(f"Directory does not exist: {directory}") return result + IGNORED_DIRS = {'.git', '__pycache__', 'node_modules', '.basic-memory'} + for path in directory.rglob("*"): - if not path.is_file() or not path.name.endswith(".md"): - if path.is_file(): - logger.debug(f"Skipping non-markdown file: {path}") + # Skip ignored directories + if path.is_dir() or path.parent.name in IGNORED_DIRS: continue try: # Get relative path first - used in error reporting if needed rel_path = str(path.relative_to(directory)) checksum = await self.file_service.compute_checksum(rel_path) + logger.debug(f"Found file: {rel_path} with checksum: {checksum}") result.files[rel_path] = checksum result.checksums[checksum] = rel_path @@ -345,8 +335,8 @@ class SyncService: result.errors[rel_path] = str(e) logger.error(f"Failed to read {rel_path}: {e}") - logger.debug(f"Found {len(result.files)} markdown files") + logger.debug(f"Found {len(result.files)} files") if result.errors: logger.warning(f"Encountered {len(result.errors)} errors while scanning") - return result \ No newline at end of file + return result diff --git a/tests/Non-MarkdownFileSupport.pdf b/tests/Non-MarkdownFileSupport.pdf new file mode 100644 index 00000000..27a4a2a3 Binary files /dev/null and b/tests/Non-MarkdownFileSupport.pdf differ diff --git a/tests/Screenshot.png b/tests/Screenshot.png new file mode 100644 index 00000000..252b076c Binary files /dev/null and b/tests/Screenshot.png differ diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index 11677bb7..c5fc7882 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -15,6 +15,34 @@ from basic_memory.services.search_service import SearchService from basic_memory.sync.sync_service import SyncService +@pytest.fixture +def test_files(test_config) -> dict[str, Path]: + """Copy test files into the project directory. + + Returns a dict mapping file names to their paths in the project dir. + """ + # Source files relative to tests directory + source_files = { + "pdf": Path("tests/Non-MarkdownFileSupport.pdf"), + "image": Path("tests/Screenshot.png") + } + + # Create copies in temp project directory + project_files = {} + for name, src_path in source_files.items(): + # Read source file + content = src_path.read_bytes() + + # Create destination path and ensure parent dirs exist + dest_path = test_config.home / src_path.name + dest_path.parent.mkdir(parents=True, exist_ok=True) + + # Write file + dest_path.write_bytes(content) + project_files[name] = dest_path + + return project_files + async def create_test_file(path: Path, content: str = "test content") -> None: """Create a test file with given content.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -842,3 +870,26 @@ test content """.strip() == file_one_content ) + + +@pytest.mark.asyncio +async def test_sync_non_markdown_files(sync_service, test_config, test_files): + """Test syncing non-markdown files.""" + report = await sync_service.sync(test_config.home) + + assert report.total == 2 + # Check files were detected + assert test_files["pdf"].name in [f for f in report.new] + assert test_files["image"].name in [f for f in report.new] + + # Verify entities were created + pdf_entity = await sync_service.entity_repository.get_by_file_path( + str(test_files["pdf"].name) + ) + assert pdf_entity is not None, "PDF entity should have been created" + assert pdf_entity.content_type == "application/pdf" + + image_entity = await sync_service.entity_repository.get_by_file_path( + str(test_files["image"].name) + ) + assert image_entity.content_type == "image/png" \ No newline at end of file