admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync non-markdown files

Browse Source
This commit is contained in:
phernandez
2025-02-22 14:50:40 -06:00
parent eb4a55a5e7
commit 20d0375ffa
6 changed files with 108 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/file_utils.py
+6 -5
View File
@@ -2,7 +2,7 @@
import hashlib
from pathlib import Path
from typing import Dict, Any
from typing import Dict, Any, Union
import yaml
from loguru import logger
@@ -26,12 +26,12 @@ class ParseError(FileError):
pass
async def compute_checksum(content: str) -> str:
async def compute_checksum(content: Union[str, bytes]) -> str:
"""
Compute SHA-256 checksum of content.
Args:
content: Text content to hash
content: Content to hash (either text string or bytes)
Returns:
SHA-256 hex digest
@@ -40,12 +40,13 @@ async def compute_checksum(content: str) -> str:
FileError: If checksum computation fails
"""
try:
return hashlib.sha256(content.encode()).hexdigest()
if isinstance(content, str):
content = content.encode()
return hashlib.sha256(content).hexdigest()
except Exception as e: # pragma: no cover
logger.error(f"Failed to compute checksum: {e}")
raise FileError(f"Failed to compute checksum: {e}")
async def ensure_directory(path: Path) -> None:
"""
Ensure directory exists, creating if necessary.
src/basic_memory/services/file_service.py
+24 -16
View File
@@ -1,4 +1,5 @@
"""Service for file operations with checksum tracking."""
import mimetypes
from os import stat_result
from pathlib import Path
@@ -7,6 +8,7 @@ from typing import Tuple, Union, Dict, Any
from loguru import logger
from basic_memory import file_utils
from basic_memory.file_utils import FileError
from basic_memory.markdown.markdown_processor import MarkdownProcessor
from basic_memory.models import Entity as EntityModel
from basic_memory.schemas import Entity as EntitySchema
@@ -185,33 +187,39 @@ class FileService:
full_path = path if path.is_absolute() else self.base_path / path
return await file_utils.update_frontmatter(full_path, updates)
async def compute_checksum(self, path: Union[Path, str]) -> str:
"""
Compute SHA-256 checksum of content.
"""
async def compute_checksum(self, path: Union[str, Path]) -> str:
"""Compute checksum for a file."""
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
# TODO checksum for binary files
return await file_utils.compute_checksum(full_path.read_text())
try:
if self.is_markdown(path):
# read str
content = await self.read_file(full_path)
else:
# read bytes
content = full_path.read_bytes()
return await file_utils.compute_checksum(content)
except Exception as e:
logger.error(f"Failed to compute checksum for {path}: {e}")
raise FileError(f"Failed to compute checksum for {path}: {e}")
def file_stats(self, path: Union[Path, str]) -> stat_result:
"""
Return file stats for a given path.
:param path:
:return:
:param path:
:return:
"""
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
# get file timestamps
return full_path.stat()
async def content_type(self, path: Union[Path, str]) -> stat_result:
def content_type(self, path: Union[Path, str]) -> str:
"""
Return content_type for a given path.
:param path:
:return:
:param path:
:return:
"""
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
@@ -220,10 +228,10 @@ class FileService:
content_type = mime_type or "text/plain"
return content_type
async def is_markdown(self, path: Union[Path, str]) -> stat_result:
def is_markdown(self, path: Union[Path, str]) -> stat_result:
"""
Return content_type for a given path.
:param path:
:return:
:param path:
:return:
"""
return self.content_type(path) == "text/markdown"
src/basic_memory/sync/sync_service.py
+27 -37
View File
@@ -3,6 +3,7 @@
import mimetypes
from dataclasses import dataclass
from dataclasses import field
from datetime import datetime
from pathlib import Path
from typing import Set, Dict, Sequence
from typing import Tuple
@@ -31,16 +32,15 @@ class SyncReport:
checksums: Current checksums for files on disk
"""
total: int = 0
# We keep paths as strings in sets/dicts for easier serialization
new: Set[str] = field(default_factory=set)
modified: Set[str] = field(default_factory=set)
deleted: Set[str] = field(default_factory=set)
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
@property
def total_changes(self) -> int:
def total(self) -> int:
"""Total number of changes."""
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
@@ -59,7 +59,6 @@ class ScanResult:
errors: Dict[str, str] = field(default_factory=dict)
class SyncService:
"""Syncs documents and knowledge files with database."""
@@ -88,10 +87,7 @@ class SyncService:
:param db_records: the data from the db
"""
db_records = await self.entity_repository.find_all()
return {
r.file_path: r.checksum or ""
for r in db_records
}
return {r.file_path: r.checksum or "" for r in db_records}
async def sync(self, directory: Path) -> SyncReport:
"""Sync all files with database."""
@@ -125,7 +121,6 @@ class SyncService:
# if checksums don't match for the same path, its modified
if local_checksum_for_db_path and db_checksum != local_checksum_for_db_path:
report.modified.add(db_path)
# check if it's moved or deleted
if not local_checksum_for_db_path:
@@ -141,11 +136,11 @@ class SyncService:
report.deleted.add(db_path)
# order of sync matters to resolve relations effectively
# sync moves first
for old_path, new_path in report.moves.items():
await self.handle_move(old_path, new_path)
# deleted next
for path in report.deleted:
await self.handle_delete(path)
@@ -196,26 +191,21 @@ class SyncService:
if new:
# Create entity with final permalink
logger.debug(f"Creating new entity from markdown: {path}")
await self.entity_service.create_entity_from_markdown(
Path(path), entity_markdown
)
await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
# otherwise we need to update the entity and observations
else:
logger.debug(f"Updating entity from markdown: {path}")
await self.entity_service.update_entity_and_observations(
Path(path), entity_markdown
)
await self.entity_service.update_entity_and_observations(Path(path), entity_markdown)
# Update relations and search index
entity = await self.entity_service.update_entity_relations(path, entity_markdown)
# set checksum
await self.entity_repository.update(entity.id, {"checksum": checksum})
return entity, checksum
async def sync_regular_file(self, path: Path, new: bool = True) -> Tuple[Entity, str]:
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
"""Sync a non-markdown file with basic tracking."""
checksum = await self.file_service.compute_checksum(path)
@@ -225,20 +215,22 @@ class SyncService:
# get file timestamps
file_stats = self.file_service.file_stats(path)
created=datetime.fromtimestamp(file_stats.st_ctime)
modified=datetime.fromtimestamp(file_stats.st_mtime)
# get mime type
mime_type, _ = mimetypes.guess_type(path.name)
content_type = mime_type or "text/plain"
content_type = self.file_service.content_type(path)
file_path = Path(path)
entity = await self.entity_repository.add(
Entity(
entity_type="file",
file_path=path,
permalink=permalink,
checksum=checksum,
title=path.name,
created_at=file_stats.st_ctime,
updated_at=file_stats.st_mtime,
title=file_path.name,
created_at=created,
updated_at=modified,
content_type=content_type,
)
)
@@ -248,7 +240,6 @@ class SyncService:
entity.id, {"file_path": path, "checksum": checksum}
)
await self.search_service.index_entity(entity)
return entity, checksum
async def handle_delete(self, file_path: str):
@@ -272,14 +263,12 @@ class SyncService:
for permalink in permalinks:
await self.search_service.delete_by_permalink(permalink)
async def handle_move(self, old_path, new_path ):
async def handle_move(self, old_path, new_path):
logger.debug(f"Moving entity: {old_path} -> {new_path}")
entity = await self.entity_repository.get_by_file_path(old_path)
if entity:
# Update file_path but keep the same permalink for link stability
updated = await self.entity_repository.update(
entity.id, {"file_path": new_path}
)
updated = await self.entity_repository.update(entity.id, {"file_path": new_path})
# update search index
await self.search_service.index_entity(updated)
@@ -312,7 +301,6 @@ class SyncService:
async def scan_directory(self, directory: Path) -> ScanResult:
"""
Scan directory for markdown files and their checksums.
Only processes .md files, logs and skips others.
Args:
directory: Directory to scan
@@ -327,16 +315,18 @@ class SyncService:
logger.debug(f"Directory does not exist: {directory}")
return result
IGNORED_DIRS = {'.git', '__pycache__', 'node_modules', '.basic-memory'}
for path in directory.rglob("*"):
if not path.is_file() or not path.name.endswith(".md"):
if path.is_file():
logger.debug(f"Skipping non-markdown file: {path}")
# Skip ignored directories
if path.is_dir() or path.parent.name in IGNORED_DIRS:
continue
try:
# Get relative path first - used in error reporting if needed
rel_path = str(path.relative_to(directory))
checksum = await self.file_service.compute_checksum(rel_path)
logger.debug(f"Found file: {rel_path} with checksum: {checksum}")
result.files[rel_path] = checksum
result.checksums[checksum] = rel_path
@@ -345,8 +335,8 @@ class SyncService:
result.errors[rel_path] = str(e)
logger.error(f"Failed to read {rel_path}: {e}")
logger.debug(f"Found {len(result.files)} markdown files")
logger.debug(f"Found {len(result.files)} files")
if result.errors:
logger.warning(f"Encountered {len(result.errors)} errors while scanning")
return result
return result
tests/Non-MarkdownFileSupport.pdf
BIN
View File
Binary file not shown.
tests/Screenshot.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 186 KiB

tests/sync/test_sync_service.py
+51
View File
@@ -15,6 +15,34 @@ from basic_memory.services.search_service import SearchService
from basic_memory.sync.sync_service import SyncService
@pytest.fixture
def test_files(test_config) -> dict[str, Path]:
"""Copy test files into the project directory.
Returns a dict mapping file names to their paths in the project dir.
"""
# Source files relative to tests directory
source_files = {
"pdf": Path("tests/Non-MarkdownFileSupport.pdf"),
"image": Path("tests/Screenshot.png")
}
# Create copies in temp project directory
project_files = {}
for name, src_path in source_files.items():
# Read source file
content = src_path.read_bytes()
# Create destination path and ensure parent dirs exist
dest_path = test_config.home / src_path.name
dest_path.parent.mkdir(parents=True, exist_ok=True)
# Write file
dest_path.write_bytes(content)
project_files[name] = dest_path
return project_files
async def create_test_file(path: Path, content: str = "test content") -> None:
"""Create a test file with given content."""
path.parent.mkdir(parents=True, exist_ok=True)
@@ -842,3 +870,26 @@ test content
""".strip()
== file_one_content
)
@pytest.mark.asyncio
async def test_sync_non_markdown_files(sync_service, test_config, test_files):
"""Test syncing non-markdown files."""
report = await sync_service.sync(test_config.home)
assert report.total == 2
# Check files were detected
assert test_files["pdf"].name in [f for f in report.new]
assert test_files["image"].name in [f for f in report.new]
# Verify entities were created
pdf_entity = await sync_service.entity_repository.get_by_file_path(
str(test_files["pdf"].name)
)
assert pdf_entity is not None, "PDF entity should have been created"
assert pdf_entity.content_type == "application/pdf"
image_entity = await sync_service.entity_repository.get_by_file_path(
str(test_files["image"].name)
)
assert image_entity.content_type == "image/png"