mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix: Sanitize filenames and allow optional kebab case (#260)
Signed-off-by: Brandon Mayes <5610870+bdmayes@users.noreply.github.com>
This commit is contained in:
Brandon Mayes
+58
-46
@@ -28,7 +28,7 @@ FilePath = Union[Path, str]
|
||||
logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def generate_permalink(file_path: Union[Path, str, PathLike]) -> str:
|
||||
def generate_permalink(file_path: Union[Path, str, PathLike], split_extension: bool = True) -> str:
|
||||
"""Generate a stable permalink from a file path.
|
||||
|
||||
Args:
|
||||
@@ -51,53 +51,59 @@ def generate_permalink(file_path: Union[Path, str, PathLike]) -> str:
|
||||
# Convert Path to string if needed
|
||||
path_str = Path(str(file_path)).as_posix()
|
||||
|
||||
# Remove extension
|
||||
base = os.path.splitext(path_str)[0]
|
||||
# Remove extension (for now, possibly)
|
||||
(base, extension) = os.path.splitext(path_str)
|
||||
|
||||
# Check if we have CJK characters that should be preserved
|
||||
# CJK ranges: \u4e00-\u9fff (CJK Unified Ideographs), \u3000-\u303f (CJK symbols),
|
||||
# CJK ranges: \u4e00-\u9fff (CJK Unified Ideographs), \u3000-\u303f (CJK symbols),
|
||||
# \u3400-\u4dbf (CJK Extension A), \uff00-\uffef (Fullwidth forms)
|
||||
has_cjk_chars = any(
|
||||
'\u4e00' <= char <= '\u9fff' or
|
||||
'\u3000' <= char <= '\u303f' or
|
||||
'\u3400' <= char <= '\u4dbf' or
|
||||
'\uff00' <= char <= '\uffef'
|
||||
"\u4e00" <= char <= "\u9fff"
|
||||
or "\u3000" <= char <= "\u303f"
|
||||
or "\u3400" <= char <= "\u4dbf"
|
||||
or "\uff00" <= char <= "\uffef"
|
||||
for char in base
|
||||
)
|
||||
|
||||
|
||||
if has_cjk_chars:
|
||||
# For text with CJK characters, selectively transliterate only Latin accented chars
|
||||
result = ""
|
||||
for char in base:
|
||||
if ('\u4e00' <= char <= '\u9fff' or
|
||||
'\u3000' <= char <= '\u303f' or
|
||||
'\u3400' <= char <= '\u4dbf'):
|
||||
if (
|
||||
"\u4e00" <= char <= "\u9fff"
|
||||
or "\u3000" <= char <= "\u303f"
|
||||
or "\u3400" <= char <= "\u4dbf"
|
||||
):
|
||||
# Preserve CJK ideographs and symbols
|
||||
result += char
|
||||
elif ('\uff00' <= char <= '\uffef'):
|
||||
elif "\uff00" <= char <= "\uffef":
|
||||
# Remove Chinese fullwidth punctuation entirely (like ，！？)
|
||||
continue
|
||||
else:
|
||||
# Transliterate Latin accented characters to ASCII
|
||||
result += unidecode(char)
|
||||
|
||||
|
||||
# Insert hyphens between CJK and Latin character transitions
|
||||
# Match: CJK followed by Latin letter/digit, or Latin letter/digit followed by CJK
|
||||
result = re.sub(r'([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])([a-zA-Z0-9])', r'\1-\2', result)
|
||||
result = re.sub(r'([a-zA-Z0-9])([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])', r'\1-\2', result)
|
||||
|
||||
result = re.sub(
|
||||
r"([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])([a-zA-Z0-9])", r"\1-\2", result
|
||||
)
|
||||
result = re.sub(
|
||||
r"([a-zA-Z0-9])([\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf])", r"\1-\2", result
|
||||
)
|
||||
|
||||
# Insert dash between camelCase
|
||||
result = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", result)
|
||||
|
||||
|
||||
# Convert ASCII letters to lowercase, preserve CJK
|
||||
lower_text = "".join(c.lower() if c.isascii() and c.isalpha() else c for c in result)
|
||||
|
||||
|
||||
# Replace underscores with hyphens
|
||||
text_with_hyphens = lower_text.replace("_", "-")
|
||||
|
||||
|
||||
# Remove apostrophes entirely (don't replace with hyphens)
|
||||
text_no_apostrophes = text_with_hyphens.replace("'", "")
|
||||
|
||||
|
||||
# Replace unsafe chars with hyphens, but preserve CJK characters
|
||||
clean_text = re.sub(
|
||||
r"[^a-z0-9\u4e00-\u9fff\u3000-\u303f\u3400-\u4dbf/\-]", "-", text_no_apostrophes
|
||||
@@ -129,7 +135,13 @@ def generate_permalink(file_path: Union[Path, str, PathLike]) -> str:
|
||||
segments = clean_text.split("/")
|
||||
clean_segments = [s.strip("-") for s in segments]
|
||||
|
||||
return "/".join(clean_segments)
|
||||
return_val = "/".join(clean_segments)
|
||||
|
||||
# Append file extension back, if necessary
|
||||
if not split_extension and extension:
|
||||
return_val += extension
|
||||
|
||||
return return_val
|
||||
|
||||
|
||||
def setup_logging(
|
||||
@@ -229,79 +241,79 @@ def normalize_newlines(multiline: str) -> str:
|
||||
Returns:
|
||||
A string with normalized newlines native to the platform.
|
||||
"""
|
||||
return re.sub(r'\r\n?|\n', os.linesep, multiline)
|
||||
return re.sub(r"\r\n?|\n", os.linesep, multiline)
|
||||
|
||||
|
||||
def normalize_file_path_for_comparison(file_path: str) -> str:
|
||||
"""Normalize a file path for conflict detection.
|
||||
|
||||
|
||||
This function normalizes file paths to help detect potential conflicts:
|
||||
- Converts to lowercase for case-insensitive comparison
|
||||
- Normalizes Unicode characters
|
||||
- Handles path separators consistently
|
||||
|
||||
|
||||
Args:
|
||||
file_path: The file path to normalize
|
||||
|
||||
|
||||
Returns:
|
||||
Normalized file path for comparison purposes
|
||||
"""
|
||||
import unicodedata
|
||||
|
||||
|
||||
# Convert to lowercase for case-insensitive comparison
|
||||
normalized = file_path.lower()
|
||||
|
||||
|
||||
# Normalize Unicode characters (NFD normalization)
|
||||
normalized = unicodedata.normalize('NFD', normalized)
|
||||
|
||||
normalized = unicodedata.normalize("NFD", normalized)
|
||||
|
||||
# Replace path separators with forward slashes
|
||||
normalized = normalized.replace('\\', '/')
|
||||
|
||||
normalized = normalized.replace("\\", "/")
|
||||
|
||||
# Remove multiple slashes
|
||||
normalized = re.sub(r'/+', '/', normalized)
|
||||
|
||||
normalized = re.sub(r"/+", "/", normalized)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def detect_potential_file_conflicts(file_path: str, existing_paths: List[str]) -> List[str]:
|
||||
"""Detect potential conflicts between a file path and existing paths.
|
||||
|
||||
|
||||
This function checks for various types of conflicts:
|
||||
- Case sensitivity differences
|
||||
- Unicode normalization differences
|
||||
- Path separator differences
|
||||
- Permalink generation conflicts
|
||||
|
||||
|
||||
Args:
|
||||
file_path: The file path to check
|
||||
existing_paths: List of existing file paths to check against
|
||||
|
||||
|
||||
Returns:
|
||||
List of existing paths that might conflict with the given file path
|
||||
"""
|
||||
conflicts = []
|
||||
|
||||
|
||||
# Normalize the input file path
|
||||
normalized_input = normalize_file_path_for_comparison(file_path)
|
||||
input_permalink = generate_permalink(file_path)
|
||||
|
||||
|
||||
for existing_path in existing_paths:
|
||||
# Skip identical paths
|
||||
if existing_path == file_path:
|
||||
continue
|
||||
|
||||
|
||||
# Check for case-insensitive path conflicts
|
||||
normalized_existing = normalize_file_path_for_comparison(existing_path)
|
||||
if normalized_input == normalized_existing:
|
||||
conflicts.append(existing_path)
|
||||
continue
|
||||
|
||||
|
||||
# Check for permalink conflicts
|
||||
existing_permalink = generate_permalink(existing_path)
|
||||
if input_permalink == existing_permalink:
|
||||
conflicts.append(existing_path)
|
||||
continue
|
||||
|
||||
|
||||
return conflicts
|
||||
|
||||
|
||||
@@ -336,13 +348,13 @@ def validate_project_path(path: str, project_path: Path) -> bool:
|
||||
|
||||
def ensure_timezone_aware(dt: datetime) -> datetime:
|
||||
"""Ensure a datetime is timezone-aware using system timezone.
|
||||
|
||||
|
||||
If the datetime is naive, convert it to timezone-aware using the system's local timezone.
|
||||
If it's already timezone-aware, return it unchanged.
|
||||
|
||||
|
||||
Args:
|
||||
dt: The datetime to ensure is timezone-aware
|
||||
|
||||
|
||||
Returns:
|
||||
A timezone-aware datetime
|
||||
"""
|
||||
@@ -351,4 +363,4 @@ def ensure_timezone_aware(dt: datetime) -> datetime:
|
||||
return dt.astimezone()
|
||||
else:
|
||||
# Already timezone-aware
|
||||
return dt
|
||||
return dt
|
||||
|
||||
Reference in New Issue
Block a user