fix: Update YAML frontmatter tag formatting for Obsidian compatibility (#280)

Signed-off-by: Drew Cain <groksrc@users.noreply.github.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: Paul Hernandez <phernandez@users.noreply.github.com>
Co-authored-by: Paul Hernandez <60959+phernandez@users.noreply.github.com>

src/basic_memory/file_utils.py

@@ -6,6 +6,7 @@ import re
 from typing import Any, Dict, Union
 
 import yaml
+import frontmatter
 from loguru import logger
 
 from basic_memory.utils import FilePath
@@ -236,6 +237,50 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str:
         raise FileError(f"Failed to update frontmatter: {e}")
 
 
+def dump_frontmatter(post: frontmatter.Post) -> str:
+    """
+    Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format.
+    
+    This function ensures that tags are formatted as YAML lists instead of JSON arrays:
+    
+    Good (Obsidian compatible):
+    ---
+    tags:
+    - system
+    - overview  
+    - reference
+    ---
+    
+    Bad (current behavior):
+    ---
+    tags: ["system", "overview", "reference"]
+    ---
+    
+    Args:
+        post: frontmatter.Post object to serialize
+        
+    Returns:
+        String containing markdown with properly formatted YAML frontmatter
+    """    
+    if not post.metadata:
+        # No frontmatter, just return content
+        return post.content
+        
+    # Serialize YAML with block style for lists
+    yaml_str = yaml.dump(
+        post.metadata,
+        sort_keys=False,
+        allow_unicode=True,
+        default_flow_style=False
+    )
+    
+    # Construct the final markdown with frontmatter
+    if post.content:
+        return f"---\n{yaml_str}---\n\n{post.content}"
+    else:
+        return f"---\n{yaml_str}---\n"
+
+
 def sanitize_for_filename(text: str, replacement: str = "-") -> str:
     """
     Sanitize string to be safe for use as a note title
@@ -252,3 +297,4 @@ def sanitize_for_filename(text: str, replacement: str = "-") -> str:
     text = re.sub(f"{re.escape(replacement)}+", replacement, text)
 
     return text.strip(replacement)
+

src/basic_memory/markdown/markdown_processor.py

@@ -7,6 +7,7 @@ from frontmatter import Post
 from loguru import logger
 
 from basic_memory import file_utils
+from basic_memory.file_utils import dump_frontmatter
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
 
@@ -115,7 +116,7 @@ class MarkdownProcessor:
 
         # Create Post object for frontmatter
         post = Post(content, **frontmatter_dict)
-        final_content = frontmatter.dumps(post, sort_keys=False)
+        final_content = dump_frontmatter(post)
 
         logger.debug(f"writing file {path} with content:\n{final_content}")
 

src/basic_memory/services/entity_service.py

@@ -9,7 +9,7 @@ from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
 from basic_memory.config import ProjectConfig, BasicMemoryConfig
-from basic_memory.file_utils import has_frontmatter, parse_frontmatter, remove_frontmatter
+from basic_memory.file_utils import has_frontmatter, parse_frontmatter, remove_frontmatter, dump_frontmatter
 from basic_memory.markdown import EntityMarkdown
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown
@@ -196,7 +196,7 @@ class EntityService(BaseService[EntityModel]):
         post = await schema_to_markdown(schema)
 
         # write file
-        final_content = frontmatter.dumps(post, sort_keys=False)
+        final_content = dump_frontmatter(post)
         checksum = await self.file_service.write_file(file_path, final_content)
 
         # parse entity from file
@@ -273,7 +273,7 @@ class EntityService(BaseService[EntityModel]):
         merged_post = frontmatter.Post(post.content, **existing_markdown.frontmatter.metadata)
 
         # write file
-        final_content = frontmatter.dumps(merged_post, sort_keys=False)
+        final_content = dump_frontmatter(merged_post)
         checksum = await self.file_service.write_file(file_path, final_content)
 
         # parse entity from file

src/basic_memory/utils.py

@@ -219,8 +219,21 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
         # First strip whitespace, then strip leading '#' characters to prevent accumulation
         return [tag.strip().lstrip("#") for tag in tags if tag and tag.strip()]
 
-    # Process comma-separated string of tags
+    # Process string input
     if isinstance(tags, str):
+        # Check if it's a JSON array string (common issue from AI assistants)
+        import json
+        if tags.strip().startswith('[') and tags.strip().endswith(']'):
+            try:
+                # Try to parse as JSON array
+                parsed_json = json.loads(tags)
+                if isinstance(parsed_json, list):
+                    # Recursively parse the JSON array as a list
+                    return parse_tags(parsed_json)
+            except json.JSONDecodeError:
+                # Not valid JSON, fall through to comma-separated parsing
+                pass
+        
         # Split by comma, strip whitespace, then strip leading '#' characters
         return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()]
 

tests/mcp/test_obsidian_yaml_formatting.py

@@ -0,0 +1,182 @@
+"""Integration tests for Obsidian-compatible YAML formatting in write_note tool."""
+
+import pytest
+from pathlib import Path
+
+from basic_memory.mcp.tools import write_note
+
+
+@pytest.mark.asyncio
+async def test_write_note_tags_yaml_format(app, project_config):
+    """Test that write_note creates files with proper YAML list format for tags."""
+    # Create a note with tags using write_note
+    result = await write_note.fn(
+        title="YAML Format Test",
+        folder="test", 
+        content="Testing YAML tag formatting",
+        tags=["system", "overview", "reference"]
+    )
+    
+    # Verify the note was created successfully
+    assert "Created note" in result
+    assert "file_path: test/YAML Format Test.md" in result
+    
+    # Read the file directly to check YAML formatting
+    file_path = project_config.home / "test" / "YAML Format Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Should use YAML list format
+    assert "tags:" in content
+    assert "- system" in content
+    assert "- overview" in content
+    assert "- reference" in content
+    
+    # Should NOT use JSON array format
+    assert '["system"' not in content
+    assert '"overview"' not in content
+    assert '"reference"]' not in content
+
+
+@pytest.mark.asyncio  
+async def test_write_note_stringified_json_tags(app, project_config):
+    """Test that stringified JSON arrays are handled correctly."""
+    # This simulates the issue where AI assistants pass tags as stringified JSON
+    result = await write_note.fn(
+        title="Stringified JSON Test",
+        folder="test",
+        content="Testing stringified JSON tag input", 
+        tags='["python", "testing", "json"]'  # Stringified JSON array
+    )
+    
+    # Verify the note was created successfully
+    assert "Created note" in result
+    
+    # Read the file to check formatting
+    file_path = project_config.home / "test" / "Stringified JSON Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Should properly parse the JSON and format as YAML list
+    assert "tags:" in content
+    assert "- python" in content
+    assert "- testing" in content
+    assert "- json" in content
+    
+    # Should NOT have the original stringified format issues
+    assert '["python"' not in content
+    assert '"testing"' not in content
+    assert '"json"]' not in content
+
+
+@pytest.mark.asyncio
+async def test_write_note_single_tag_yaml_format(app, project_config):
+    """Test that single tags are still formatted as YAML lists."""
+    result = await write_note.fn(
+        title="Single Tag Test",
+        folder="test",
+        content="Testing single tag formatting",
+        tags=["solo-tag"]
+    )
+    
+    file_path = project_config.home / "test" / "Single Tag Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Single tag should still use list format
+    assert "tags:" in content
+    assert "- solo-tag" in content
+
+
+@pytest.mark.asyncio
+async def test_write_note_no_tags(app, project_config):
+    """Test that notes without tags work normally."""
+    result = await write_note.fn(
+        title="No Tags Test",
+        folder="test", 
+        content="Testing note without tags",
+        tags=None
+    )
+    
+    file_path = project_config.home / "test" / "No Tags Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Should not have tags field in frontmatter
+    assert "tags:" not in content
+    assert "title: No Tags Test" in content
+
+
+@pytest.mark.asyncio
+async def test_write_note_empty_tags_list(app, project_config):
+    """Test that empty tag lists are handled properly."""
+    result = await write_note.fn(
+        title="Empty Tags Test",
+        folder="test",
+        content="Testing empty tag list",
+        tags=[]
+    )
+    
+    file_path = project_config.home / "test" / "Empty Tags Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Should not add tags field to frontmatter for empty lists
+    assert "tags:" not in content
+    
+
+@pytest.mark.asyncio  
+async def test_write_note_update_preserves_yaml_format(app, project_config):
+    """Test that updating a note preserves the YAML list format."""
+    # First, create the note
+    await write_note.fn(
+        title="Update Format Test",
+        folder="test",
+        content="Initial content",
+        tags=["initial", "tag"]
+    )
+    
+    # Then update it with new tags
+    result = await write_note.fn(
+        title="Update Format Test", 
+        folder="test",
+        content="Updated content", 
+        tags=["updated", "new-tag", "format"]
+    )
+    
+    # Should be an update, not a new creation
+    assert "Updated note" in result
+    
+    # Check the file format
+    file_path = project_config.home / "test" / "Update Format Test.md" 
+    content = file_path.read_text(encoding="utf-8")
+    
+    # Should have proper YAML formatting for updated tags
+    assert "tags:" in content
+    assert "- updated" in content
+    assert "- new-tag" in content
+    assert "- format" in content
+    
+    # Old tags should be gone
+    assert "- initial" not in content
+    assert "- tag" not in content
+    
+    # Content should be updated
+    assert "Updated content" in content
+    assert "Initial content" not in content
+
+
+@pytest.mark.asyncio
+async def test_complex_tags_yaml_format(app, project_config):
+    """Test that complex tags with special characters format correctly."""
+    result = await write_note.fn(
+        title="Complex Tags Test",
+        folder="test",
+        content="Testing complex tag formats",
+        tags=["python-3.9", "api_integration", "v2.0", "nested/category", "under_score"]
+    )
+    
+    file_path = project_config.home / "test" / "Complex Tags Test.md"
+    content = file_path.read_text(encoding="utf-8")
+    
+    # All complex tags should format correctly
+    assert "- python-3.9" in content
+    assert "- api_integration" in content  
+    assert "- v2.0" in content
+    assert "- nested/category" in content
+    assert "- under_score" in content

tests/utils/test_frontmatter_obsidian_compatible.py

@@ -0,0 +1,184 @@
+"""Tests for Obsidian-compatible YAML frontmatter formatting."""
+
+import pytest
+import frontmatter
+
+from basic_memory.file_utils import dump_frontmatter
+
+
+def test_tags_formatted_as_yaml_list():
+    """Test that tags are formatted as YAML list instead of JSON array."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["type"] = "note"
+    post.metadata["tags"] = ["system", "overview", "reference"]
+    
+    result = dump_frontmatter(post)
+    
+    # Should use YAML list format
+    assert "tags:" in result
+    assert "- system" in result
+    assert "- overview" in result  
+    assert "- reference" in result
+    
+    # Should NOT use JSON array format
+    assert '["system"' not in result
+    assert '"overview"' not in result
+    assert '"reference"]' not in result
+
+
+def test_empty_tags_list():
+    """Test that empty tags list is handled correctly."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["tags"] = []
+    
+    result = dump_frontmatter(post)
+    
+    # Should have empty list representation
+    assert "tags: []" in result
+
+
+def test_single_tag():
+    """Test that single tag is still formatted as list."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["tags"] = ["single-tag"]
+    
+    result = dump_frontmatter(post)
+    
+    assert "tags:" in result
+    assert "- single-tag" in result
+    
+
+def test_no_tags_metadata():
+    """Test that posts without tags work normally."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["type"] = "note"
+    
+    result = dump_frontmatter(post)
+    
+    assert "title: Test Note" in result
+    assert "type: note" in result
+    assert "tags:" not in result
+
+
+def test_no_frontmatter():
+    """Test that posts with no frontmatter just return content."""
+    post = frontmatter.Post("Test content only")
+    
+    result = dump_frontmatter(post)
+    
+    assert result == "Test content only"
+
+
+def test_complex_tags_with_special_characters():
+    """Test tags with hyphens, underscores, and other valid characters."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["tags"] = ["python-test", "api_integration", "v2.0", "nested/tag"]
+    
+    result = dump_frontmatter(post)
+    
+    assert "- python-test" in result
+    assert "- api_integration" in result
+    assert "- v2.0" in result
+    assert "- nested/tag" in result
+
+
+def test_tags_order_preserved():
+    """Test that tag order is preserved in output."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["tags"] = ["zebra", "apple", "banana"]
+    
+    result = dump_frontmatter(post)
+    
+    # Find the positions of each tag in the output
+    zebra_pos = result.find("- zebra")
+    apple_pos = result.find("- apple") 
+    banana_pos = result.find("- banana")
+    
+    # They should appear in the same order as input
+    assert zebra_pos < apple_pos < banana_pos
+
+
+def test_non_tags_lists_also_formatted():
+    """Test that other lists in metadata are also formatted properly."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["authors"] = ["John Doe", "Jane Smith"]
+    post.metadata["keywords"] = ["AI", "machine learning"]
+    
+    result = dump_frontmatter(post)
+    
+    # Authors should be formatted as YAML list
+    assert "authors:" in result
+    assert "- John Doe" in result
+    assert "- Jane Smith" in result
+    
+    # Keywords should be formatted as YAML list
+    assert "keywords:" in result
+    assert "- AI" in result
+    assert "- machine learning" in result
+
+
+def test_mixed_metadata_types():
+    """Test that mixed metadata types are handled correctly."""
+    post = frontmatter.Post("Test content")
+    post.metadata["title"] = "Test Note"
+    post.metadata["tags"] = ["tag1", "tag2"] 
+    post.metadata["created"] = "2024-01-01"
+    post.metadata["priority"] = 5
+    post.metadata["draft"] = True
+    
+    result = dump_frontmatter(post)
+    
+    # Lists should use YAML format
+    assert "tags:" in result
+    assert "- tag1" in result
+    assert "- tag2" in result
+    
+    # Other types should be normal
+    assert "title: Test Note" in result
+    assert "created: '2024-01-01'" in result or "created: 2024-01-01" in result
+    assert "priority: 5" in result
+    assert "draft: true" in result or "draft: True" in result
+
+
+def test_empty_content():
+    """Test posts with empty content but with frontmatter."""
+    post = frontmatter.Post("")
+    post.metadata["title"] = "Empty Note"
+    post.metadata["tags"] = ["empty", "test"]
+    
+    result = dump_frontmatter(post)
+    
+    # Should have frontmatter delimiter
+    assert result.startswith("---")
+    assert result.endswith("---\n")
+    
+    # Should have proper tag formatting
+    assert "- empty" in result
+    assert "- test" in result
+
+
+def test_roundtrip_compatibility():
+    """Test that the formatted output can be parsed back by frontmatter."""
+    original_post = frontmatter.Post("Test content")
+    original_post.metadata["title"] = "Test Note"
+    original_post.metadata["tags"] = ["system", "test", "obsidian"]
+    original_post.metadata["type"] = "note"
+    
+    # Format with our function
+    formatted = dump_frontmatter(original_post)
+    
+    # Parse it back
+    parsed_post = frontmatter.loads(formatted)
+    
+    # Should have same content and metadata
+    assert parsed_post.content == original_post.content
+    assert parsed_post.metadata["title"] == original_post.metadata["title"]
+    assert parsed_post.metadata["tags"] == original_post.metadata["tags"]
+    assert parsed_post.metadata["type"] == original_post.metadata["type"]

tests/utils/test_parse_tags.py

@@ -30,6 +30,11 @@ from basic_memory.utils import parse_tags
         # Mixed whitespace and '#' characters
         ([" #tag1 ", " ##tag2 "], ["tag1", "tag2"]),
         (" #tag1 , ##tag2 ", ["tag1", "tag2"]),
+        # JSON stringified arrays (common AI assistant issue)
+        ('["tag1", "tag2", "tag3"]', ["tag1", "tag2", "tag3"]),
+        ('["system", "overview", "reference"]', ["system", "overview", "reference"]),
+        ('["#tag1", "##tag2"]', ["tag1", "tag2"]),  # JSON array with hash prefixes
+        ('[ "tag1" , "tag2" ]', ["tag1", "tag2"]),  # JSON array with extra spaces
     ],
 )
 def test_parse_tags(input_tags: Union[List[str], str, None], expected: List[str]) -> None:
@@ -48,3 +53,16 @@ def test_parse_tags_special_case() -> None:
 
     result = parse_tags(TagObject())  # pyright: ignore [reportArgumentType]
     assert result == ["tag1", "tag2"]
+
+
+def test_parse_tags_invalid_json() -> None:
+    """Test that invalid JSON strings fall back to comma-separated parsing."""
+    # Invalid JSON should fall back to comma-separated parsing
+    result = parse_tags('[invalid json')
+    assert result == ["[invalid json"]  # Treated as single tag
+    
+    result = parse_tags('[tag1, tag2]')  # Valid bracket format but not JSON
+    assert result == ["[tag1", "tag2]"]  # Split by comma
+    
+    result = parse_tags('["tag1", "tag2"')  # Incomplete JSON
+    assert result == ['["tag1"', '"tag2"']  # Fall back to comma separation