From 22f7bfa3989179bb2e9e76aef191e7d3ffd67d31 Mon Sep 17 00:00:00 2001 From: Drew Cain Date: Thu, 4 Sep 2025 09:57:45 -0500 Subject: [PATCH] fix: Update YAML frontmatter tag formatting for Obsidian compatibility (#280) Signed-off-by: Drew Cain Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: Paul Hernandez Co-authored-by: Paul Hernandez <60959+phernandez@users.noreply.github.com> --- src/basic_memory/file_utils.py | 46 +++++ .../markdown/markdown_processor.py | 3 +- src/basic_memory/services/entity_service.py | 6 +- src/basic_memory/utils.py | 15 +- tests/mcp/test_obsidian_yaml_formatting.py | 182 +++++++++++++++++ .../test_frontmatter_obsidian_compatible.py | 184 ++++++++++++++++++ tests/utils/test_parse_tags.py | 18 ++ 7 files changed, 449 insertions(+), 5 deletions(-) create mode 100644 tests/mcp/test_obsidian_yaml_formatting.py create mode 100644 tests/utils/test_frontmatter_obsidian_compatible.py diff --git a/src/basic_memory/file_utils.py b/src/basic_memory/file_utils.py index 5605fdfa..3161cdc4 100644 --- a/src/basic_memory/file_utils.py +++ b/src/basic_memory/file_utils.py @@ -6,6 +6,7 @@ import re from typing import Any, Dict, Union import yaml +import frontmatter from loguru import logger from basic_memory.utils import FilePath @@ -236,6 +237,50 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str: raise FileError(f"Failed to update frontmatter: {e}") +def dump_frontmatter(post: frontmatter.Post) -> str: + """ + Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format. + + This function ensures that tags are formatted as YAML lists instead of JSON arrays: + + Good (Obsidian compatible): + --- + tags: + - system + - overview + - reference + --- + + Bad (current behavior): + --- + tags: ["system", "overview", "reference"] + --- + + Args: + post: frontmatter.Post object to serialize + + Returns: + String containing markdown with properly formatted YAML frontmatter + """ + if not post.metadata: + # No frontmatter, just return content + return post.content + + # Serialize YAML with block style for lists + yaml_str = yaml.dump( + post.metadata, + sort_keys=False, + allow_unicode=True, + default_flow_style=False + ) + + # Construct the final markdown with frontmatter + if post.content: + return f"---\n{yaml_str}---\n\n{post.content}" + else: + return f"---\n{yaml_str}---\n" + + def sanitize_for_filename(text: str, replacement: str = "-") -> str: """ Sanitize string to be safe for use as a note title @@ -252,3 +297,4 @@ def sanitize_for_filename(text: str, replacement: str = "-") -> str: text = re.sub(f"{re.escape(replacement)}+", replacement, text) return text.strip(replacement) + diff --git a/src/basic_memory/markdown/markdown_processor.py b/src/basic_memory/markdown/markdown_processor.py index cadca6d8..77d5d4dc 100644 --- a/src/basic_memory/markdown/markdown_processor.py +++ b/src/basic_memory/markdown/markdown_processor.py @@ -7,6 +7,7 @@ from frontmatter import Post from loguru import logger from basic_memory import file_utils +from basic_memory.file_utils import dump_frontmatter from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation @@ -115,7 +116,7 @@ class MarkdownProcessor: # Create Post object for frontmatter post = Post(content, **frontmatter_dict) - final_content = frontmatter.dumps(post, sort_keys=False) + final_content = dump_frontmatter(post) logger.debug(f"writing file {path} with content:\n{final_content}") diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 7ded793a..94985dd4 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -9,7 +9,7 @@ from loguru import logger from sqlalchemy.exc import IntegrityError from basic_memory.config import ProjectConfig, BasicMemoryConfig -from basic_memory.file_utils import has_frontmatter, parse_frontmatter, remove_frontmatter +from basic_memory.file_utils import has_frontmatter, parse_frontmatter, remove_frontmatter, dump_frontmatter from basic_memory.markdown import EntityMarkdown from basic_memory.markdown.entity_parser import EntityParser from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown @@ -196,7 +196,7 @@ class EntityService(BaseService[EntityModel]): post = await schema_to_markdown(schema) # write file - final_content = frontmatter.dumps(post, sort_keys=False) + final_content = dump_frontmatter(post) checksum = await self.file_service.write_file(file_path, final_content) # parse entity from file @@ -273,7 +273,7 @@ class EntityService(BaseService[EntityModel]): merged_post = frontmatter.Post(post.content, **existing_markdown.frontmatter.metadata) # write file - final_content = frontmatter.dumps(merged_post, sort_keys=False) + final_content = dump_frontmatter(merged_post) checksum = await self.file_service.write_file(file_path, final_content) # parse entity from file diff --git a/src/basic_memory/utils.py b/src/basic_memory/utils.py index 4bb4a812..1df9f43d 100644 --- a/src/basic_memory/utils.py +++ b/src/basic_memory/utils.py @@ -219,8 +219,21 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]: # First strip whitespace, then strip leading '#' characters to prevent accumulation return [tag.strip().lstrip("#") for tag in tags if tag and tag.strip()] - # Process comma-separated string of tags + # Process string input if isinstance(tags, str): + # Check if it's a JSON array string (common issue from AI assistants) + import json + if tags.strip().startswith('[') and tags.strip().endswith(']'): + try: + # Try to parse as JSON array + parsed_json = json.loads(tags) + if isinstance(parsed_json, list): + # Recursively parse the JSON array as a list + return parse_tags(parsed_json) + except json.JSONDecodeError: + # Not valid JSON, fall through to comma-separated parsing + pass + # Split by comma, strip whitespace, then strip leading '#' characters return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()] diff --git a/tests/mcp/test_obsidian_yaml_formatting.py b/tests/mcp/test_obsidian_yaml_formatting.py new file mode 100644 index 00000000..26ac03bc --- /dev/null +++ b/tests/mcp/test_obsidian_yaml_formatting.py @@ -0,0 +1,182 @@ +"""Integration tests for Obsidian-compatible YAML formatting in write_note tool.""" + +import pytest +from pathlib import Path + +from basic_memory.mcp.tools import write_note + + +@pytest.mark.asyncio +async def test_write_note_tags_yaml_format(app, project_config): + """Test that write_note creates files with proper YAML list format for tags.""" + # Create a note with tags using write_note + result = await write_note.fn( + title="YAML Format Test", + folder="test", + content="Testing YAML tag formatting", + tags=["system", "overview", "reference"] + ) + + # Verify the note was created successfully + assert "Created note" in result + assert "file_path: test/YAML Format Test.md" in result + + # Read the file directly to check YAML formatting + file_path = project_config.home / "test" / "YAML Format Test.md" + content = file_path.read_text(encoding="utf-8") + + # Should use YAML list format + assert "tags:" in content + assert "- system" in content + assert "- overview" in content + assert "- reference" in content + + # Should NOT use JSON array format + assert '["system"' not in content + assert '"overview"' not in content + assert '"reference"]' not in content + + +@pytest.mark.asyncio +async def test_write_note_stringified_json_tags(app, project_config): + """Test that stringified JSON arrays are handled correctly.""" + # This simulates the issue where AI assistants pass tags as stringified JSON + result = await write_note.fn( + title="Stringified JSON Test", + folder="test", + content="Testing stringified JSON tag input", + tags='["python", "testing", "json"]' # Stringified JSON array + ) + + # Verify the note was created successfully + assert "Created note" in result + + # Read the file to check formatting + file_path = project_config.home / "test" / "Stringified JSON Test.md" + content = file_path.read_text(encoding="utf-8") + + # Should properly parse the JSON and format as YAML list + assert "tags:" in content + assert "- python" in content + assert "- testing" in content + assert "- json" in content + + # Should NOT have the original stringified format issues + assert '["python"' not in content + assert '"testing"' not in content + assert '"json"]' not in content + + +@pytest.mark.asyncio +async def test_write_note_single_tag_yaml_format(app, project_config): + """Test that single tags are still formatted as YAML lists.""" + result = await write_note.fn( + title="Single Tag Test", + folder="test", + content="Testing single tag formatting", + tags=["solo-tag"] + ) + + file_path = project_config.home / "test" / "Single Tag Test.md" + content = file_path.read_text(encoding="utf-8") + + # Single tag should still use list format + assert "tags:" in content + assert "- solo-tag" in content + + +@pytest.mark.asyncio +async def test_write_note_no_tags(app, project_config): + """Test that notes without tags work normally.""" + result = await write_note.fn( + title="No Tags Test", + folder="test", + content="Testing note without tags", + tags=None + ) + + file_path = project_config.home / "test" / "No Tags Test.md" + content = file_path.read_text(encoding="utf-8") + + # Should not have tags field in frontmatter + assert "tags:" not in content + assert "title: No Tags Test" in content + + +@pytest.mark.asyncio +async def test_write_note_empty_tags_list(app, project_config): + """Test that empty tag lists are handled properly.""" + result = await write_note.fn( + title="Empty Tags Test", + folder="test", + content="Testing empty tag list", + tags=[] + ) + + file_path = project_config.home / "test" / "Empty Tags Test.md" + content = file_path.read_text(encoding="utf-8") + + # Should not add tags field to frontmatter for empty lists + assert "tags:" not in content + + +@pytest.mark.asyncio +async def test_write_note_update_preserves_yaml_format(app, project_config): + """Test that updating a note preserves the YAML list format.""" + # First, create the note + await write_note.fn( + title="Update Format Test", + folder="test", + content="Initial content", + tags=["initial", "tag"] + ) + + # Then update it with new tags + result = await write_note.fn( + title="Update Format Test", + folder="test", + content="Updated content", + tags=["updated", "new-tag", "format"] + ) + + # Should be an update, not a new creation + assert "Updated note" in result + + # Check the file format + file_path = project_config.home / "test" / "Update Format Test.md" + content = file_path.read_text(encoding="utf-8") + + # Should have proper YAML formatting for updated tags + assert "tags:" in content + assert "- updated" in content + assert "- new-tag" in content + assert "- format" in content + + # Old tags should be gone + assert "- initial" not in content + assert "- tag" not in content + + # Content should be updated + assert "Updated content" in content + assert "Initial content" not in content + + +@pytest.mark.asyncio +async def test_complex_tags_yaml_format(app, project_config): + """Test that complex tags with special characters format correctly.""" + result = await write_note.fn( + title="Complex Tags Test", + folder="test", + content="Testing complex tag formats", + tags=["python-3.9", "api_integration", "v2.0", "nested/category", "under_score"] + ) + + file_path = project_config.home / "test" / "Complex Tags Test.md" + content = file_path.read_text(encoding="utf-8") + + # All complex tags should format correctly + assert "- python-3.9" in content + assert "- api_integration" in content + assert "- v2.0" in content + assert "- nested/category" in content + assert "- under_score" in content \ No newline at end of file diff --git a/tests/utils/test_frontmatter_obsidian_compatible.py b/tests/utils/test_frontmatter_obsidian_compatible.py new file mode 100644 index 00000000..a8c234f2 --- /dev/null +++ b/tests/utils/test_frontmatter_obsidian_compatible.py @@ -0,0 +1,184 @@ +"""Tests for Obsidian-compatible YAML frontmatter formatting.""" + +import pytest +import frontmatter + +from basic_memory.file_utils import dump_frontmatter + + +def test_tags_formatted_as_yaml_list(): + """Test that tags are formatted as YAML list instead of JSON array.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["type"] = "note" + post.metadata["tags"] = ["system", "overview", "reference"] + + result = dump_frontmatter(post) + + # Should use YAML list format + assert "tags:" in result + assert "- system" in result + assert "- overview" in result + assert "- reference" in result + + # Should NOT use JSON array format + assert '["system"' not in result + assert '"overview"' not in result + assert '"reference"]' not in result + + +def test_empty_tags_list(): + """Test that empty tags list is handled correctly.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["tags"] = [] + + result = dump_frontmatter(post) + + # Should have empty list representation + assert "tags: []" in result + + +def test_single_tag(): + """Test that single tag is still formatted as list.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["tags"] = ["single-tag"] + + result = dump_frontmatter(post) + + assert "tags:" in result + assert "- single-tag" in result + + +def test_no_tags_metadata(): + """Test that posts without tags work normally.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["type"] = "note" + + result = dump_frontmatter(post) + + assert "title: Test Note" in result + assert "type: note" in result + assert "tags:" not in result + + +def test_no_frontmatter(): + """Test that posts with no frontmatter just return content.""" + post = frontmatter.Post("Test content only") + + result = dump_frontmatter(post) + + assert result == "Test content only" + + +def test_complex_tags_with_special_characters(): + """Test tags with hyphens, underscores, and other valid characters.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["tags"] = ["python-test", "api_integration", "v2.0", "nested/tag"] + + result = dump_frontmatter(post) + + assert "- python-test" in result + assert "- api_integration" in result + assert "- v2.0" in result + assert "- nested/tag" in result + + +def test_tags_order_preserved(): + """Test that tag order is preserved in output.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["tags"] = ["zebra", "apple", "banana"] + + result = dump_frontmatter(post) + + # Find the positions of each tag in the output + zebra_pos = result.find("- zebra") + apple_pos = result.find("- apple") + banana_pos = result.find("- banana") + + # They should appear in the same order as input + assert zebra_pos < apple_pos < banana_pos + + +def test_non_tags_lists_also_formatted(): + """Test that other lists in metadata are also formatted properly.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["authors"] = ["John Doe", "Jane Smith"] + post.metadata["keywords"] = ["AI", "machine learning"] + + result = dump_frontmatter(post) + + # Authors should be formatted as YAML list + assert "authors:" in result + assert "- John Doe" in result + assert "- Jane Smith" in result + + # Keywords should be formatted as YAML list + assert "keywords:" in result + assert "- AI" in result + assert "- machine learning" in result + + +def test_mixed_metadata_types(): + """Test that mixed metadata types are handled correctly.""" + post = frontmatter.Post("Test content") + post.metadata["title"] = "Test Note" + post.metadata["tags"] = ["tag1", "tag2"] + post.metadata["created"] = "2024-01-01" + post.metadata["priority"] = 5 + post.metadata["draft"] = True + + result = dump_frontmatter(post) + + # Lists should use YAML format + assert "tags:" in result + assert "- tag1" in result + assert "- tag2" in result + + # Other types should be normal + assert "title: Test Note" in result + assert "created: '2024-01-01'" in result or "created: 2024-01-01" in result + assert "priority: 5" in result + assert "draft: true" in result or "draft: True" in result + + +def test_empty_content(): + """Test posts with empty content but with frontmatter.""" + post = frontmatter.Post("") + post.metadata["title"] = "Empty Note" + post.metadata["tags"] = ["empty", "test"] + + result = dump_frontmatter(post) + + # Should have frontmatter delimiter + assert result.startswith("---") + assert result.endswith("---\n") + + # Should have proper tag formatting + assert "- empty" in result + assert "- test" in result + + +def test_roundtrip_compatibility(): + """Test that the formatted output can be parsed back by frontmatter.""" + original_post = frontmatter.Post("Test content") + original_post.metadata["title"] = "Test Note" + original_post.metadata["tags"] = ["system", "test", "obsidian"] + original_post.metadata["type"] = "note" + + # Format with our function + formatted = dump_frontmatter(original_post) + + # Parse it back + parsed_post = frontmatter.loads(formatted) + + # Should have same content and metadata + assert parsed_post.content == original_post.content + assert parsed_post.metadata["title"] == original_post.metadata["title"] + assert parsed_post.metadata["tags"] == original_post.metadata["tags"] + assert parsed_post.metadata["type"] == original_post.metadata["type"] \ No newline at end of file diff --git a/tests/utils/test_parse_tags.py b/tests/utils/test_parse_tags.py index f451077d..b226effe 100644 --- a/tests/utils/test_parse_tags.py +++ b/tests/utils/test_parse_tags.py @@ -30,6 +30,11 @@ from basic_memory.utils import parse_tags # Mixed whitespace and '#' characters ([" #tag1 ", " ##tag2 "], ["tag1", "tag2"]), (" #tag1 , ##tag2 ", ["tag1", "tag2"]), + # JSON stringified arrays (common AI assistant issue) + ('["tag1", "tag2", "tag3"]', ["tag1", "tag2", "tag3"]), + ('["system", "overview", "reference"]', ["system", "overview", "reference"]), + ('["#tag1", "##tag2"]', ["tag1", "tag2"]), # JSON array with hash prefixes + ('[ "tag1" , "tag2" ]', ["tag1", "tag2"]), # JSON array with extra spaces ], ) def test_parse_tags(input_tags: Union[List[str], str, None], expected: List[str]) -> None: @@ -48,3 +53,16 @@ def test_parse_tags_special_case() -> None: result = parse_tags(TagObject()) # pyright: ignore [reportArgumentType] assert result == ["tag1", "tag2"] + + +def test_parse_tags_invalid_json() -> None: + """Test that invalid JSON strings fall back to comma-separated parsing.""" + # Invalid JSON should fall back to comma-separated parsing + result = parse_tags('[invalid json') + assert result == ["[invalid json"] # Treated as single tag + + result = parse_tags('[tag1, tag2]') # Valid bracket format but not JSON + assert result == ["[tag1", "tag2]"] # Split by comma + + result = parse_tags('["tag1", "tag2"') # Incomplete JSON + assert result == ['["tag1"', '"tag2"'] # Fall back to comma separation