mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Signed-off-by: phernandez <paul@basicmachines.co> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Paul Hernandez
@@ -4,7 +4,7 @@ Uses markdown-it with plugins to parse structured data from markdown content.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -26,6 +26,82 @@ from basic_memory.utils import parse_tags
|
||||
md = MarkdownIt().use(observation_plugin).use(relation_plugin)
|
||||
|
||||
|
||||
def normalize_frontmatter_value(value: Any) -> Any:
|
||||
"""Normalize frontmatter values to safe types for processing.
|
||||
|
||||
PyYAML automatically converts various string-like values into native Python types:
|
||||
- Date strings ("2025-10-24") → datetime.date objects
|
||||
- Numbers ("1.0") → int or float
|
||||
- Booleans ("true") → bool
|
||||
- Lists → list objects
|
||||
|
||||
This can cause AttributeError when code expects strings and calls string methods
|
||||
like .strip() on these values (see GitHub issue #236).
|
||||
|
||||
This function normalizes all frontmatter values to safe types:
|
||||
- Dates/datetimes → ISO format strings
|
||||
- Numbers (int/float) → strings
|
||||
- Booleans → strings ("True"/"False")
|
||||
- Lists → preserved as lists, but items are recursively normalized
|
||||
- Dicts → preserved as dicts, but values are recursively normalized
|
||||
- Strings → kept as-is
|
||||
- None → kept as None
|
||||
|
||||
Args:
|
||||
value: The frontmatter value to normalize
|
||||
|
||||
Returns:
|
||||
The normalized value safe for string operations
|
||||
|
||||
Example:
|
||||
>>> normalize_frontmatter_value(datetime.date(2025, 10, 24))
|
||||
'2025-10-24'
|
||||
>>> normalize_frontmatter_value([datetime.date(2025, 10, 24), "tag", 123])
|
||||
['2025-10-24', 'tag', '123']
|
||||
>>> normalize_frontmatter_value(True)
|
||||
'True'
|
||||
"""
|
||||
# Convert date/datetime objects to ISO format strings
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, date):
|
||||
return value.isoformat()
|
||||
|
||||
# Convert boolean to string (must come before int check since bool is subclass of int)
|
||||
if isinstance(value, bool):
|
||||
return str(value)
|
||||
|
||||
# Convert numbers to strings
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
|
||||
# Recursively process lists (preserve as list, normalize items)
|
||||
if isinstance(value, list):
|
||||
return [normalize_frontmatter_value(item) for item in value]
|
||||
|
||||
# Recursively process dicts (preserve as dict, normalize values)
|
||||
if isinstance(value, dict):
|
||||
return {key: normalize_frontmatter_value(val) for key, val in value.items()}
|
||||
|
||||
# Keep strings and None as-is
|
||||
return value
|
||||
|
||||
|
||||
def normalize_frontmatter_metadata(metadata: dict) -> dict:
|
||||
"""Normalize all values in frontmatter metadata dict.
|
||||
|
||||
Converts date/datetime objects to ISO format strings to prevent
|
||||
AttributeError when code expects strings (GitHub issue #236).
|
||||
|
||||
Args:
|
||||
metadata: The frontmatter metadata dictionary
|
||||
|
||||
Returns:
|
||||
A new dictionary with all values normalized
|
||||
"""
|
||||
return {key: normalize_frontmatter_value(value) for key, value in metadata.items()}
|
||||
|
||||
|
||||
@dataclass
|
||||
class EntityContent:
|
||||
content: str
|
||||
@@ -127,20 +203,25 @@ class EntityParser:
|
||||
|
||||
# Extract file stat info
|
||||
file_stats = absolute_path.stat()
|
||||
metadata = post.metadata
|
||||
|
||||
# Normalize frontmatter values to prevent AttributeError on date objects (issue #236)
|
||||
# PyYAML automatically converts date strings like "2025-10-24" to datetime.date objects
|
||||
# This normalization converts them back to ISO format strings to ensure compatibility
|
||||
# with code that expects string values
|
||||
metadata = normalize_frontmatter_metadata(post.metadata)
|
||||
|
||||
# Ensure required fields have defaults (issue #184, #387)
|
||||
# Handle title - use default if missing, None/null, empty, or string "None"
|
||||
title = post.metadata.get("title")
|
||||
title = metadata.get("title")
|
||||
if not title or title == "None":
|
||||
metadata["title"] = absolute_path.stem
|
||||
else:
|
||||
metadata["title"] = title
|
||||
# Handle type - use default if missing OR explicitly set to None/null
|
||||
entity_type = post.metadata.get("type")
|
||||
entity_type = metadata.get("type")
|
||||
metadata["type"] = entity_type if entity_type is not None else "note"
|
||||
|
||||
tags = parse_tags(post.metadata.get("tags", [])) # pyright: ignore
|
||||
tags = parse_tags(metadata.get("tags", [])) # pyright: ignore
|
||||
if tags:
|
||||
metadata["tags"] = tags
|
||||
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Test that YAML date parsing doesn't break frontmatter processing.
|
||||
|
||||
This test reproduces GitHub issue #236 from basic-memory-cloud where date fields
|
||||
in YAML frontmatter are automatically parsed as datetime.date objects by PyYAML,
|
||||
but later code expects strings and calls .strip() on them, causing AttributeError.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from basic_memory.markdown.entity_parser import EntityParser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_file_with_date(tmp_path):
|
||||
"""Create a test file with date fields in frontmatter."""
|
||||
test_file = tmp_path / "test_note.md"
|
||||
content = """---
|
||||
title: Test Note
|
||||
date: 2025-10-24
|
||||
created: 2025-10-24
|
||||
tags:
|
||||
- python
|
||||
- testing
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
This file has date fields in frontmatter that PyYAML will parse as datetime.date objects.
|
||||
"""
|
||||
test_file.write_text(content)
|
||||
return test_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_file_with_date_in_tags(tmp_path):
|
||||
"""Create a test file with a date value in tags (edge case)."""
|
||||
test_file = tmp_path / "test_note_date_tags.md"
|
||||
content = """---
|
||||
title: Test Note with Date Tags
|
||||
tags: 2025-10-24
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
This file has a date value as tags, which will be parsed as datetime.date.
|
||||
"""
|
||||
test_file.write_text(content)
|
||||
return test_file
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_file_with_dates_in_tag_list(tmp_path):
|
||||
"""Create a test file with dates in a tag list (edge case)."""
|
||||
test_file = tmp_path / "test_note_dates_in_list.md"
|
||||
content = """---
|
||||
title: Test Note with Dates in Tags List
|
||||
tags:
|
||||
- valid-tag
|
||||
- 2025-10-24
|
||||
- another-tag
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
This file has date values mixed into tags list.
|
||||
"""
|
||||
test_file.write_text(content)
|
||||
return test_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_file_with_date_fields(test_file_with_date, tmp_path):
|
||||
"""Test that files with date fields in frontmatter can be parsed without errors."""
|
||||
parser = EntityParser(tmp_path)
|
||||
|
||||
# This should not raise AttributeError about .strip()
|
||||
entity_markdown = await parser.parse_file(test_file_with_date)
|
||||
|
||||
# Verify basic parsing worked
|
||||
assert entity_markdown.frontmatter.title == "Test Note"
|
||||
|
||||
# Date fields should be converted to ISO format strings
|
||||
date_field = entity_markdown.frontmatter.metadata.get("date")
|
||||
assert date_field is not None
|
||||
assert isinstance(date_field, str), "Date should be converted to string"
|
||||
assert date_field == "2025-10-24", "Date should be in ISO format"
|
||||
|
||||
created_field = entity_markdown.frontmatter.metadata.get("created")
|
||||
assert created_field is not None
|
||||
assert isinstance(created_field, str), "Created date should be converted to string"
|
||||
assert created_field == "2025-10-24", "Created date should be in ISO format"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_file_with_date_as_tags(test_file_with_date_in_tags, tmp_path):
|
||||
"""Test that date values in tags field don't cause errors."""
|
||||
parser = EntityParser(tmp_path)
|
||||
|
||||
# This should not raise AttributeError - date should be converted to string
|
||||
entity_markdown = await parser.parse_file(test_file_with_date_in_tags)
|
||||
assert entity_markdown.frontmatter.title == "Test Note with Date Tags"
|
||||
|
||||
# The date should be converted to ISO format string before parse_tags processes it
|
||||
tags = entity_markdown.frontmatter.tags
|
||||
assert tags is not None
|
||||
assert isinstance(tags, list)
|
||||
# The date value should be converted to string
|
||||
assert "2025-10-24" in tags
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_file_with_dates_in_tag_list(test_file_with_dates_in_tag_list, tmp_path):
|
||||
"""Test that date values in a tags list don't cause errors."""
|
||||
parser = EntityParser(tmp_path)
|
||||
|
||||
# This should not raise AttributeError - dates should be converted to strings
|
||||
entity_markdown = await parser.parse_file(test_file_with_dates_in_tag_list)
|
||||
assert entity_markdown.frontmatter.title == "Test Note with Dates in Tags List"
|
||||
|
||||
# Tags should be parsed
|
||||
tags = entity_markdown.frontmatter.tags
|
||||
assert tags is not None
|
||||
assert isinstance(tags, list)
|
||||
|
||||
# Should have 3 tags (2 valid + 1 date converted to ISO string)
|
||||
assert len(tags) == 3
|
||||
assert "valid-tag" in tags
|
||||
assert "another-tag" in tags
|
||||
# Date should be converted to ISO format string
|
||||
assert "2025-10-24" in tags
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_file_with_various_yaml_types(tmp_path):
|
||||
"""Test that various YAML types in frontmatter don't cause errors.
|
||||
|
||||
This reproduces the broader issue from GitHub #236 where ANY non-string
|
||||
YAML type (dates, lists, numbers, booleans) can cause AttributeError
|
||||
when code expects strings and calls .strip().
|
||||
"""
|
||||
test_file = tmp_path / "test_yaml_types.md"
|
||||
content = """---
|
||||
title: Test YAML Types
|
||||
date: 2025-10-24
|
||||
priority: 1
|
||||
completed: true
|
||||
tags:
|
||||
- python
|
||||
- testing
|
||||
metadata:
|
||||
author: Test User
|
||||
version: 1.0
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
This file has various YAML types that need to be normalized.
|
||||
"""
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity_markdown = await parser.parse_file(test_file)
|
||||
|
||||
# All values should be accessible without AttributeError
|
||||
assert entity_markdown.frontmatter.title == "Test YAML Types"
|
||||
|
||||
# Date should be converted to ISO string
|
||||
date_field = entity_markdown.frontmatter.metadata.get("date")
|
||||
assert isinstance(date_field, str)
|
||||
assert date_field == "2025-10-24"
|
||||
|
||||
# Number should be converted to string
|
||||
priority = entity_markdown.frontmatter.metadata.get("priority")
|
||||
assert isinstance(priority, str)
|
||||
assert priority == "1"
|
||||
|
||||
# Boolean should be converted to string
|
||||
completed = entity_markdown.frontmatter.metadata.get("completed")
|
||||
assert isinstance(completed, str)
|
||||
assert completed == "True" # Python's str(True) always returns "True"
|
||||
|
||||
# List should be preserved as list, but items should be strings
|
||||
tags = entity_markdown.frontmatter.tags
|
||||
assert isinstance(tags, list)
|
||||
assert all(isinstance(tag, str) for tag in tags)
|
||||
assert "python" in tags
|
||||
assert "testing" in tags
|
||||
|
||||
# Dict should be preserved as dict, but nested values should be strings
|
||||
metadata = entity_markdown.frontmatter.metadata.get("metadata")
|
||||
assert isinstance(metadata, dict)
|
||||
assert isinstance(metadata.get("author"), str)
|
||||
assert metadata.get("author") == "Test User"
|
||||
assert isinstance(metadata.get("version"), str)
|
||||
assert metadata.get("version") in ("1.0", "1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_file_with_datetime_objects(tmp_path):
|
||||
"""Test that datetime objects (not just date objects) are properly normalized.
|
||||
|
||||
This tests the edge case where frontmatter might contain datetime values
|
||||
with time components (as parsed by PyYAML), ensuring they're converted to ISO format strings.
|
||||
"""
|
||||
test_file = tmp_path / "test_datetime.md"
|
||||
|
||||
# YAML datetime strings that PyYAML will parse as datetime objects
|
||||
# Format: YYYY-MM-DD HH:MM:SS or YYYY-MM-DDTHH:MM:SS
|
||||
content = """---
|
||||
title: Test Datetime
|
||||
created_at: 2025-10-24 14:30:00
|
||||
updated_at: 2025-10-24T00:00:00
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
This file has datetime values in frontmatter that PyYAML will parse as datetime objects.
|
||||
"""
|
||||
test_file.write_text(content)
|
||||
|
||||
parser = EntityParser(tmp_path)
|
||||
entity_markdown = await parser.parse_file(test_file)
|
||||
|
||||
# Verify datetime objects are converted to ISO format strings
|
||||
created_at = entity_markdown.frontmatter.metadata.get("created_at")
|
||||
assert isinstance(created_at, str), "Datetime should be converted to string"
|
||||
# PyYAML parses "2025-10-24 14:30:00" as datetime, which we normalize to ISO
|
||||
assert "2025-10-24" in created_at and "14:30:00" in created_at, \
|
||||
f"Datetime with time should be normalized to ISO format, got: {created_at}"
|
||||
|
||||
updated_at = entity_markdown.frontmatter.metadata.get("updated_at")
|
||||
assert isinstance(updated_at, str), "Datetime should be converted to string"
|
||||
# PyYAML parses "2025-10-24T00:00:00" as datetime, which we normalize to ISO
|
||||
assert "2025-10-24" in updated_at and "00:00:00" in updated_at, \
|
||||
f"Datetime at midnight should be normalized to ISO format, got: {updated_at}"
|
||||
Reference in New Issue
Block a user