From e9829000849168978af5e1dc860af4e45c7ec140 Mon Sep 17 00:00:00 2001
From: phernandez <paul@basicmachines.co>
Date: Mon, 6 Apr 2026 23:14:40 -0500
Subject: [PATCH] fix(core): strip null bytes from markdown content before
 database insert

PostgreSQL rejects null bytes (0x00) in text columns, causing
CharacterNotInRepertoireError when syncing files like Claude agent
definitions that contain embedded nulls. SQLite silently accepts them,
so this only surfaces in cloud environments.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>
---
 src/basic_memory/markdown/entity_parser.py |  4 ++++
 tests/markdown/test_parser_edge_cases.py   | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py
index 0259df92..f8676dd9 100644
--- a/src/basic_memory/markdown/entity_parser.py
+++ b/src/basic_memory/markdown/entity_parser.py
@@ -249,6 +249,10 @@ class EntityParser:
 
         content = strip_bom(content)
 
+        # PostgreSQL rejects null bytes (0x00) in text columns.
+        # Some markdown files (e.g. Claude agent definitions) contain embedded nulls.
+        content = content.replace("\x00", "")
+
         # Parse frontmatter with proper error handling for malformed YAML.
         # We use frontmatter.parse() instead of frontmatter.loads() because
         # loads() does Post(content, handler, **metadata), which crashes when
diff --git a/tests/markdown/test_parser_edge_cases.py b/tests/markdown/test_parser_edge_cases.py
index 32a961c6..00ae5d23 100644
--- a/tests/markdown/test_parser_edge_cases.py
+++ b/tests/markdown/test_parser_edge_cases.py
@@ -176,6 +176,27 @@ async def test_malformed_frontmatter(tmp_path):
     assert entity.frontmatter.permalink is None
 
 
+@pytest.mark.asyncio
+async def test_null_bytes_stripped(tmp_path):
+    """Test that null bytes are stripped from content before parsing.
+
+    PostgreSQL rejects null bytes (0x00) in text columns. Some files
+    (e.g. Claude agent definitions) can contain embedded nulls.
+    """
+    content = "---\ntitle: Test\ntype: note\n---\n\nSome content\x00with nulls\x00inside\n"
+
+    parser = EntityParser(tmp_path)
+    entity = await parser.parse_markdown_content(
+        file_path=tmp_path / "nulls.md",
+        content=content,
+    )
+
+    assert "\x00" not in entity.content
+    assert "Some content" in entity.content
+    assert "with nulls" in entity.content
+    assert "inside" in entity.content
+
+
 @pytest.mark.asyncio
 async def test_file_not_found():
     """Test handling of non-existent files."""