diff --git a/src/basic_memory/deps.py b/src/basic_memory/deps.py index ee63e2b2..18147371 100644 --- a/src/basic_memory/deps.py +++ b/src/basic_memory/deps.py @@ -12,7 +12,6 @@ from sqlalchemy.ext.asyncio import ( from basic_memory import db from basic_memory.config import ProjectConfig, config from basic_memory.markdown.knowledge_writer import KnowledgeWriter -from basic_memory.markdown.note_writer import NoteWriter from basic_memory.repository.entity_repository import EntityRepository from basic_memory.repository.observation_repository import ObservationRepository from basic_memory.repository.relation_repository import RelationRepository @@ -172,11 +171,6 @@ async def get_knowledge_writer() -> KnowledgeWriter: KnowledgeWriterDep = Annotated[KnowledgeWriter, Depends(get_knowledge_writer)] -async def get_note_writer() -> NoteWriter: - return NoteWriter() - - -NoteWriterDep = Annotated[NoteWriter, Depends(get_note_writer)] async def get_knowledge_service( @@ -185,7 +179,6 @@ async def get_knowledge_service( relation_service: RelationServiceDep, file_service: FileServiceDep, knowledge_writer: KnowledgeWriterDep, - note_writer: NoteWriterDep, project_config: ProjectConfigDep, ) -> KnowledgeService: """Create KnowledgeService with dependencies.""" @@ -195,7 +188,6 @@ async def get_knowledge_service( relation_service=relation_service, file_service=file_service, knowledge_writer=knowledge_writer, - note_writer=note_writer, base_path=project_config.knowledge_dir, ) diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py new file mode 100644 index 00000000..4e04c615 --- /dev/null +++ b/src/basic_memory/markdown/entity_parser.py @@ -0,0 +1,317 @@ +"""Universal parser for markdown files with optional frontmatter, observations, and relations. + +The id field in frontmatter is derived from the filename, converted to snake_case with .md extension removed. +For example: + 'My Project Notes.md' -> 'my_project_notes' + 'API-Design.md' -> 'api_design' +""" + +import re +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, Optional, Tuple, List + +from loguru import logger + +from basic_memory.markdown.base_parser import MarkdownParser, ParseError +from basic_memory.markdown.schemas import ( + EntityMarkdown, + EntityFrontmatter, + EntityContent, + EntityMetadata, + Observation, + Relation, +) +from basic_memory.schemas.base import to_snake_case + + +class EntityParser(MarkdownParser[EntityMarkdown]): + """A forgiving parser that extracts as much structure as it can find. + + Generates entity IDs from filenames by: + 1. Removing .md extension + 2. Converting to snake_case + 3. Removing any invalid characters + """ + + def convert_to_id(self, filename: str) -> str: + """Convert a filename to a valid entity ID. + + Args: + filename: Name of the file (with or without .md extension) + + Returns: + Snake case version of filename without extension + + Examples: + 'My Project Notes.md' -> 'my_project_notes' + 'API-Design.md' -> 'api_design' + """ + # Remove .md extension if present + if filename.lower().endswith('.md'): + filename = filename[:-3] + + return to_snake_case(filename) + + def parse_dates(self, frontmatter: Dict[str, Any], file_path: Path) -> Tuple[datetime, datetime]: + """Parse created and updated dates from frontmatter or file system. + + Args: + frontmatter: Dictionary containing frontmatter fields + file_path: Path to the source file for fallback dates + + Returns: + Tuple of (created_date, updated_date) + + Priority: + 1. Valid frontmatter dates + 2. File system dates (created/modified) + """ + created = None + updated = None + + # Try frontmatter first + try: + if 'created' in frontmatter: + created = self.parse_date(frontmatter['created']) + if 'updated' in frontmatter or 'modified' in frontmatter: + updated = self.parse_date(frontmatter.get('updated') or frontmatter.get('modified')) + except Exception as e: + logger.warning(f"Error parsing frontmatter dates: {e}") + + # Fall back to file system dates if needed + try: + stats = file_path.stat() + if not created: + created = datetime.fromtimestamp(stats.st_ctime) + if not updated: + updated = datetime.fromtimestamp(stats.st_mtime) + except Exception as e: + logger.warning(f"Error getting file stats: {e}") + # Last resort - use current time + now = datetime.now() + created = created or now + updated = updated or now + + return created, updated + + def parse_date(self, value: Any) -> Optional[datetime]: + """Convert various date formats to datetime.""" + if isinstance(value, datetime): + return value + try: + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except (ValueError, TypeError): + pass + return None + + def parse_tags(self, tags: Any) -> List[str]: + """Convert various tag formats to list of strings.""" + if isinstance(tags, str): + return [t.strip() for t in tags.split(",") if t.strip()] + if isinstance(tags, (list, tuple)): + return [str(t).strip() for t in tags if str(t).strip()] + return [] + + async def parse_frontmatter(self, frontmatter: Dict[str, Any], file_path: Optional[Path] = None) -> EntityFrontmatter: + """Parse frontmatter with sensible defaults for missing fields. + + Args: + frontmatter: Dictionary of frontmatter fields + file_path: Optional path to source file, used for id and dates + """ + try: + # Get or generate ID from filename + entity_name = None + if file_path: + entity_name = self.convert_to_id(file_path.name) + + # Get dates from frontmatter or file + created, updated = self.parse_dates(frontmatter, file_path) if file_path else (datetime.now(), datetime.now()) + + # Ensure we have minimum required fields + processed = { + "type": str(frontmatter.get("type", "document")).strip(), + "id": str(frontmatter.get("id", entity_name or "document")).strip(), + "created": created, + "modified": updated, + "tags": self.parse_tags(frontmatter.get("tags", [])) + } + + return EntityFrontmatter(**processed) + + except Exception as e: + logger.warning(f"Error parsing frontmatter, using defaults: {e}") + return EntityFrontmatter( + type="document", + id=entity_name or "document", + created=created, + modified=updated, + tags=[] + ) + + async def parse_content(self, title: str, sections: Dict[str, str]) -> EntityContent: + """Parse content sections without requiring any particular structure.""" + try: + # Get content from content section if it exists + content = sections.get("content", "").strip() or None + + # Try to parse observations if they exist + observations = [] + if "observations" in sections: + for line in sections["observations"].splitlines(): + try: + obs = await self.parse_observation(line) + if obs: + observations.append(obs) + except ParseError as e: + logger.warning(f"Skipping invalid observation: {e}") + + # Try to parse relations if they exist + relations = [] + if "relations" in sections: + for line in sections["relations"].splitlines(): + try: + rel = await self.parse_relation(line) + if rel: + relations.append(rel) + except ParseError as e: + logger.warning(f"Skipping invalid relation: {e}") + + # Also look for wiki-links in content as implicit relations + try: + content_relations = await self.parse_content_relations(sections.get("content", "")) + relations.extend(content_relations) + except Exception as e: + logger.warning(f"Error parsing content relations: {e}") + + return EntityContent( + title=title or "Untitled", + summary=content, + observations=observations, + relations=relations + ) + + except Exception as e: + logger.error(f"Error parsing content, using minimal structure: {e}") + return EntityContent( + title=title or "Untitled", + summary=None, + observations=[], + relations=[] + ) + + async def parse_observation(self, line: str) -> Optional[Observation]: + """Parse a single observation line.""" + if not line or not line.strip().startswith("-"): + return None + + line = line.strip()[1:].strip() # Remove leading "-" and whitespace + + # Extract category if present + category = None + if line.startswith("["): + end = line.find("]") + if end != -1: + category = line[1:end].strip() + line = line[end + 1:].strip() + + # Extract context if present + context = None + if line.endswith(")"): + start = line.rfind("(") + if start != -1: + context = line[start + 1:-1].strip() + line = line[:start].strip() + + # Extract tags and content + parts = line.split() + content_parts = [] + tags = [] + + for part in parts: + if part.startswith("#"): + tags.append(part[1:]) + else: + content_parts.append(part) + + content = " ".join(content_parts).strip() + if not content: + return None + + return Observation( + category=category, + content=content, + tags=tags if tags else None, + context=context + ) + + async def parse_relation(self, line: str) -> Optional[Relation]: + """Parse a single relation line.""" + if not line or not line.strip().startswith("-"): + return None + + line = line.strip()[1:].strip() + + # Look for [[target]] + start = line.find("[[") + end = line.find("]]") + if start == -1 or end == -1: + return None + + # Extract parts + rel_type = line[:start].strip() or "relates_to" # Default type if none specified + target = line[start + 2:end].strip() + + # Extract context if present + context = None + remaining = line[end + 2:].strip() + if remaining.startswith("(") and remaining.endswith(")"): + context = remaining[1:-1].strip() + + if not target: + return None + + return Relation( + type=rel_type, + target=target, + context=context + ) + + async def parse_content_relations(self, content: str) -> List[Relation]: + """Extract wiki-style links from content as relations.""" + relations = [] + if not content: + return relations + + import re + pattern = r'\[\[([^\]]+)\]\]' + + for match in re.finditer(pattern, content): + target = match.group(1).strip() + if target: + relations.append(Relation( + type="mentions", + target=target, + context=None + )) + + return relations + + async def parse_metadata(self, metadata_section: Optional[str]) -> EntityMetadata: + """Metadata section is no longer used.""" + return EntityMetadata() + + async def create_document( + self, + frontmatter: EntityFrontmatter, + content: EntityContent, + metadata: EntityMetadata + ) -> EntityMarkdown: + """Create the final EntityMarkdown document.""" + return EntityMarkdown( + frontmatter=frontmatter, + content=content, + entity_metadata=metadata + ) \ No newline at end of file diff --git a/src/basic_memory/services/knowledge/file_operations.py b/src/basic_memory/services/knowledge/file_operations.py index 8210a628..60f86c8d 100644 --- a/src/basic_memory/services/knowledge/file_operations.py +++ b/src/basic_memory/services/knowledge/file_operations.py @@ -6,7 +6,6 @@ from typing import Tuple, Optional from loguru import logger from basic_memory.markdown.knowledge_writer import KnowledgeWriter -from basic_memory.markdown.note_writer import NoteWriter from basic_memory.models import Entity as EntityModel from basic_memory.services.entity_service import EntityService from basic_memory.services.exceptions import FileOperationError @@ -21,13 +20,11 @@ class FileOperations: entity_service: EntityService, file_service: FileService, knowledge_writer: KnowledgeWriter, - note_writer: NoteWriter, base_path: Path, ): self.entity_service = entity_service self.file_service = file_service self.knowledge_writer = knowledge_writer - self.note_writer = note_writer self.base_path = base_path async def file_exists(self, path: Path) -> bool: diff --git a/src/basic_memory/services/knowledge/knowledge_service.py b/src/basic_memory/services/knowledge/knowledge_service.py index a247afff..f34054e3 100644 --- a/src/basic_memory/services/knowledge/knowledge_service.py +++ b/src/basic_memory/services/knowledge/knowledge_service.py @@ -5,7 +5,6 @@ from typing import List, Sequence, Tuple, Dict, Any, Optional from basic_memory.markdown.knowledge_writer import KnowledgeWriter -from basic_memory.markdown.note_writer import NoteWriter from basic_memory.models import Entity as EntityModel from basic_memory.schemas import Entity as EntitySchema from basic_memory.schemas import Relation as RelationSchema @@ -42,7 +41,6 @@ class KnowledgeService: relation_service: RelationService, file_service: FileService, knowledge_writer: KnowledgeWriter, - note_writer: NoteWriter, base_path: Path, ): self.base_path = base_path @@ -52,7 +50,6 @@ class KnowledgeService: entity_service=entity_service, file_service=file_service, knowledge_writer=knowledge_writer, - note_writer=note_writer, base_path=base_path, ) diff --git a/tests/cli/test_status.py b/tests/cli/test_status.py index 3735aaf3..b419d6a7 100644 --- a/tests/cli/test_status.py +++ b/tests/cli/test_status.py @@ -7,7 +7,8 @@ import pytest_asyncio from rich.console import Console from basic_memory.cli.commands.status import display_changes, run_status -from basic_memory.sync.utils import SyncReport, FileState +from basic_memory.sync.file_change_scanner import FileState +from basic_memory.sync.utils import SyncReport from basic_memory.utils.file_utils import compute_checksum diff --git a/tests/conftest.py b/tests/conftest.py index 183a20a4..121d1f1f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,6 @@ from basic_memory.config import ProjectConfig from basic_memory.db import DatabaseType from basic_memory.markdown.knowledge_parser import KnowledgeParser from basic_memory.markdown.knowledge_writer import KnowledgeWriter -from basic_memory.markdown.note_writer import NoteWriter from basic_memory.models import Base from basic_memory.models.knowledge import Entity from basic_memory.repository.entity_repository import EntityRepository @@ -129,11 +128,6 @@ def knowledge_writer(): return KnowledgeWriter() -@pytest.fixture -def note_writer(): - """Create writer instance.""" - return NoteWriter() - @pytest.fixture def knowledge_parser(): @@ -154,7 +148,6 @@ async def knowledge_service( relation_service: RelationService, file_service: FileService, knowledge_writer: KnowledgeWriter, - note_writer: NoteWriter, test_config: ProjectConfig, ) -> KnowledgeService: """Create KnowledgeService with dependencies.""" @@ -164,7 +157,6 @@ async def knowledge_service( relation_service=relation_service, file_service=file_service, knowledge_writer=knowledge_writer, - note_writer=note_writer, base_path=test_config.knowledge_dir, ) diff --git a/tests/sync/test_file_change_scanner.py b/tests/sync/test_file_change_scanner.py index 9b419c63..520e68ff 100644 --- a/tests/sync/test_file_change_scanner.py +++ b/tests/sync/test_file_change_scanner.py @@ -3,10 +3,11 @@ from pathlib import Path import pytest +from multipart import file_path from basic_memory.models import Entity from basic_memory.sync import FileChangeScanner -from basic_memory.sync.utils import FileState +from basic_memory.sync.file_change_scanner import FileState from basic_memory.utils.file_utils import compute_checksum @@ -45,9 +46,9 @@ async def test_scan_with_mixed_files(file_change_scanner: FileChangeScanner, tem assert len(result.errors) == 0 # Verify FileState objects - assert isinstance(result.files["doc.md"], FileState) - assert result.files["doc.md"].path_id == "doc.md" - assert result.files["doc.md"].checksum is not None + assert isinstance(result.files["doc.md"], str) + # checksum + assert result.files["doc.md"] is not None @pytest.mark.asyncio @@ -84,35 +85,35 @@ async def test_detect_new_files( @pytest.mark.asyncio async def test_detect_modified_file(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test detection of modified files.""" - path = "test.md" + file_path = "test.md" content = "original" - await create_test_file(temp_dir / path, content) + await create_test_file(temp_dir / file_path, content) # Create DB state with original checksum original_checksum = await compute_checksum(content) - db_records = {path: FileState(path_id=path, checksum=original_checksum)} + db_records = {file_path: FileState(file_path=file_path, path_id="test", checksum=original_checksum)} # Modify file - await create_test_file(temp_dir / path, "modified") + await create_test_file(temp_dir / file_path, "modified") changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) assert len(changes.modified) == 1 - assert path in changes.modified + assert file_path in changes.modified @pytest.mark.asyncio async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp_dir: Path): """Test detection of deleted files.""" - path = "deleted.md" + file_path = "deleted.md" # Create DB state with file that doesn't exist - db_records = {path: FileState(path_id=path, checksum="any-checksum")} + db_records = {file_path: FileState(file_path=file_path, path_id="deleted", checksum="any-checksum")} changes = await file_change_scanner.find_changes(directory=temp_dir, db_file_state=db_records) assert len(changes.deleted) == 1 - assert path in changes.deleted + assert file_path in changes.deleted