mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 238490e37d | |||
| 825418c712 |
@@ -72,6 +72,11 @@ class BasicMemoryConfig(BaseSettings):
|
||||
description="Whether to sync changes in real time. default (True)",
|
||||
)
|
||||
|
||||
filename_format: Literal["original", "kebab-case"] = Field(
|
||||
default="kebab-case",
|
||||
description="Format for generated filenames. 'original' preserves spaces and special chars, 'kebab-case' converts them to hyphens for consistency with permalinks",
|
||||
)
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="BASIC_MEMORY_",
|
||||
extra="ignore",
|
||||
|
||||
@@ -22,7 +22,7 @@ from dateparser import parse
|
||||
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
from basic_memory.utils import generate_permalink
|
||||
from basic_memory.utils import generate_permalink, sanitize_filename
|
||||
|
||||
|
||||
def to_snake_case(name: str) -> str:
|
||||
@@ -187,10 +187,20 @@ class Entity(BaseModel):
|
||||
@property
|
||||
def file_path(self):
|
||||
"""Get the file path for this entity based on its permalink."""
|
||||
# Import here to avoid circular dependency
|
||||
try:
|
||||
from basic_memory.config import app_config
|
||||
use_kebab_case = app_config.filename_format == "kebab-case"
|
||||
except ImportError:
|
||||
# Fallback to original behavior if config not available
|
||||
use_kebab_case = True
|
||||
|
||||
filename = sanitize_filename(self.title) if use_kebab_case else self.title
|
||||
|
||||
if self.content_type == "text/markdown":
|
||||
return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
|
||||
return f"{self.folder}/{filename}.md" if self.folder else f"{filename}.md"
|
||||
else:
|
||||
return f"{self.folder}/{self.title}" if self.folder else self.title
|
||||
return f"{self.folder}/{filename}" if self.folder else filename
|
||||
|
||||
@property
|
||||
def permalink(self) -> Permalink:
|
||||
|
||||
@@ -27,6 +27,96 @@ FilePath = Union[Path, str]
|
||||
logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def sanitize_filename(title: str) -> str:
|
||||
"""
|
||||
Sanitize a title to create a safe filename.
|
||||
|
||||
Converts forward slashes and other problematic characters to hyphens
|
||||
to prevent unintended directory creation and ensure consistency
|
||||
with permalink generation.
|
||||
|
||||
Args:
|
||||
title: The original title
|
||||
|
||||
Returns:
|
||||
Sanitized filename safe for use as a file name
|
||||
|
||||
Examples:
|
||||
>>> sanitize_filename("Coupon Enable/Disable Feature")
|
||||
'coupon-enable-disable-feature'
|
||||
>>> sanitize_filename("My Awesome Feature")
|
||||
'my-awesome-feature'
|
||||
>>> sanitize_filename("Test_File Name.txt")
|
||||
'test-file-name-txt'
|
||||
"""
|
||||
# Create a transliteration mapping for specific characters
|
||||
transliteration_map = {
|
||||
"ø": "o", # Handle Søren -> soren
|
||||
"å": "a", # Handle Kierkegård -> kierkegard
|
||||
"ü": "u", # Handle Müller -> muller
|
||||
"é": "e", # Handle Café -> cafe
|
||||
"è": "e", # Handle Mère -> mere
|
||||
"ê": "e", # Handle Fête -> fete
|
||||
"à": "a", # Handle À la mode -> a la mode
|
||||
"ç": "c", # Handle Façade -> facade
|
||||
"ñ": "n", # Handle Niño -> nino
|
||||
"ö": "o", # Handle Björk -> bjork
|
||||
"ä": "a", # Handle Häagen -> haagen
|
||||
}
|
||||
|
||||
# Process character by character, transliterating Latin characters with diacritics
|
||||
result = ""
|
||||
for char in title:
|
||||
# Direct mapping for known characters
|
||||
if char.lower() in transliteration_map:
|
||||
result += transliteration_map[char.lower()]
|
||||
# General case using Unicode normalization
|
||||
elif unicodedata.category(char).startswith("L") and ord(char) > 127:
|
||||
# Decompose the character (e.g., ü -> u + combining diaeresis)
|
||||
decomposed = unicodedata.normalize("NFD", char)
|
||||
# If decomposition produced multiple characters and first one is ASCII
|
||||
if len(decomposed) > 1 and ord(decomposed[0]) < 128:
|
||||
# Keep only the base character
|
||||
result += decomposed[0].lower()
|
||||
else:
|
||||
# For non-Latin scripts like Chinese, preserve the character
|
||||
result += char
|
||||
else:
|
||||
# Add the character as is
|
||||
result += char
|
||||
|
||||
# Handle special punctuation cases for apostrophes
|
||||
result = result.replace("'", "")
|
||||
|
||||
# Insert dash between camelCase
|
||||
result = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", result)
|
||||
|
||||
# Insert dash between Chinese and Latin character boundaries
|
||||
result = re.sub(r"([\u4e00-\u9fff])([a-zA-Z])", r"\1-\2", result)
|
||||
result = re.sub(r"([a-zA-Z])([\u4e00-\u9fff])", r"\1-\2", result)
|
||||
|
||||
# Convert ASCII letters to lowercase, preserve non-ASCII characters
|
||||
lower_text = "".join(c.lower() if c.isascii() and c.isalpha() else c for c in result)
|
||||
|
||||
# Replace underscores with hyphens
|
||||
text_with_hyphens = lower_text.replace("_", "-")
|
||||
|
||||
# Replace spaces, forward slashes, and unsafe ASCII characters with hyphens
|
||||
# Include common Chinese character ranges and other non-ASCII characters
|
||||
clean_text = re.sub(
|
||||
r"[^a-z0-9\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf\-]", "-", text_with_hyphens
|
||||
)
|
||||
|
||||
# Collapse multiple hyphens
|
||||
clean_text = re.sub(r"-+", "-", clean_text)
|
||||
|
||||
# Remove hyphens between adjacent Chinese characters only
|
||||
clean_text = re.sub(r"([\u4e00-\u9fff])-([\u4e00-\u9fff])", r"\1\2", clean_text)
|
||||
|
||||
# Remove leading and trailing hyphens
|
||||
return clean_text.strip("-")
|
||||
|
||||
|
||||
def generate_permalink(file_path: Union[Path, str, Any]) -> str:
|
||||
"""
|
||||
Generate a permalink from a file path.
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.services import EntityService
|
||||
from basic_memory.sync.sync_service import SyncService
|
||||
from basic_memory.utils import generate_permalink
|
||||
from basic_memory.utils import generate_permalink, sanitize_filename
|
||||
|
||||
|
||||
async def create_test_file(path: Path, content: str = "test content") -> None:
|
||||
@@ -118,3 +118,92 @@ def test_chinese_character_preservation(input_path, expected):
|
||||
def test_mixed_character_sets(input_path, expected):
|
||||
"""Test handling of mixed character sets and edge cases."""
|
||||
assert generate_permalink(input_path) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_title, expected",
|
||||
[
|
||||
("Coupon Enable/Disable Feature", "coupon-enable-disable-feature"),
|
||||
("My Awesome Feature", "my-awesome-feature"),
|
||||
("Test_File Name.txt", "test-file-name-txt"),
|
||||
("Feature/With/Multiple/Slashes", "feature-with-multiple-slashes"),
|
||||
("Mixed Case Feature", "mixed-case-feature"),
|
||||
(" Leading and trailing spaces ", "leading-and-trailing-spaces"),
|
||||
("Special!@#$%^&*()Characters", "special-characters"),
|
||||
("北京/东京", "北京-东京"),
|
||||
("Mixed/中文/English", "mixed-中文-english"),
|
||||
],
|
||||
)
|
||||
def test_sanitize_filename(input_title, expected):
|
||||
"""Test that title sanitization works correctly for filenames."""
|
||||
assert sanitize_filename(input_title) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entity_file_path_with_kebab_case_config(
|
||||
sync_service: SyncService, project_config: ProjectConfig, entity_service: EntityService
|
||||
):
|
||||
"""Test that Entity file_path uses kebab-case when configured."""
|
||||
# Test with kebab-case enabled (default)
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.config import app_config
|
||||
|
||||
# Ensure config is set to kebab-case
|
||||
original_format = app_config.filename_format
|
||||
app_config.filename_format = "kebab-case"
|
||||
|
||||
try:
|
||||
entity = Entity(
|
||||
title="Coupon Enable/Disable Feature",
|
||||
folder="bugs",
|
||||
content="Test content"
|
||||
)
|
||||
|
||||
# With kebab-case, forward slashes should be converted to hyphens
|
||||
expected_file_path = "bugs/coupon-enable-disable-feature.md"
|
||||
expected_permalink = "bugs/coupon-enable-disable-feature"
|
||||
|
||||
assert entity.file_path == expected_file_path
|
||||
assert entity.permalink == expected_permalink
|
||||
|
||||
# Test that file_path and permalink are now consistent
|
||||
assert entity.permalink == generate_permalink(entity.file_path)
|
||||
|
||||
finally:
|
||||
# Restore original configuration
|
||||
app_config.filename_format = original_format
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_entity_file_path_with_original_config(
|
||||
sync_service: SyncService, project_config: ProjectConfig, entity_service: EntityService
|
||||
):
|
||||
"""Test that Entity file_path preserves original format when configured."""
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.config import app_config
|
||||
|
||||
# Set config to original format
|
||||
original_format = app_config.filename_format
|
||||
app_config.filename_format = "original"
|
||||
|
||||
try:
|
||||
entity = Entity(
|
||||
title="Coupon Enable/Disable Feature",
|
||||
folder="bugs",
|
||||
content="Test content"
|
||||
)
|
||||
|
||||
# With original format, spaces and slashes are preserved
|
||||
# This creates the inconsistency that the issue reports
|
||||
expected_file_path = "bugs/Coupon Enable/Disable Feature.md"
|
||||
expected_permalink = "bugs/coupon-enable-disable-feature"
|
||||
|
||||
assert entity.file_path == expected_file_path
|
||||
assert entity.permalink == expected_permalink
|
||||
|
||||
# This demonstrates the inconsistency when using original format
|
||||
assert entity.permalink != generate_permalink(entity.file_path)
|
||||
|
||||
finally:
|
||||
# Restore original configuration
|
||||
app_config.filename_format = original_format
|
||||
|
||||
Reference in New Issue
Block a user