admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use relative file paths in importers for cloud storage compatibility

Browse Source 
Importers now use relative file paths (based on permalink) instead of
absolute paths. This enables proper S3 key generation in cloud environments.

Changes:
- write_entity() accepts str | Path for file_path parameter
- ensure_folder_exists() uses relative paths directly
- All importers pass relative paths to FileService
- FileService handles base_path resolution internally

This prevents S3 keys from including container filesystem paths like
`/app/basic-memory/imports/...` and instead uses clean relative paths
like `imports/20241010-file.md`.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
phernandez
2025-12-30 20:29:01 -06:00
parent 45ce1813e4
commit 8adf1f4ed4
6 changed files with 78 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/importers/base.py
+5 -10
View File
@@ -55,7 +55,7 @@ class Importer[T: ImportResult]:
"""
pass # pragma: no cover
async def write_entity(self, entity: EntityMarkdown, file_path: Path) -> str:
async def write_entity(self, entity: EntityMarkdown, file_path: str | Path) -> str:
"""Write entity to file using FileService.
This method serializes the entity to markdown and writes it using
@@ -64,7 +64,7 @@ class Importer[T: ImportResult]:
Args:
entity: EntityMarkdown instance to write.
file_path: Path to write the entity to.
file_path: Relative path to write the entity to. FileService handles base_path.
Returns:
Checksum of written file.
@@ -73,21 +73,16 @@ class Importer[T: ImportResult]:
# FileService.write_file handles directory creation and returns checksum
return await self.file_service.write_file(file_path, content)
async def ensure_folder_exists(self, folder: str) -> Path:
async def ensure_folder_exists(self, folder: str) -> None:
"""Ensure folder exists using FileService.
For cloud storage (S3), this is essentially a no-op since S3 doesn't
have actual folders - they're just key prefixes.
Args:
folder: Folder name or path within the project.
Returns:
Path to the folder.
folder: Relative folder path within the project. FileService handles base_path.
"""
folder_path = self.base_path / folder
await self.file_service.ensure_directory(folder_path)
return folder_path
await self.file_service.ensure_directory(folder)
@abstractmethod
def handle_error(
src/basic_memory/importers/chatgpt_importer.py
+2 -2
View File
@@ -41,8 +41,8 @@ class ChatGPTImporter(Importer[ChatImportResult]):
# Convert to entity
entity = self._format_chat_content(destination_folder, chat)
# Write file
file_path = self.base_path / f"{entity.frontmatter.metadata['permalink']}.md"
# Write file using relative path - FileService handles base_path
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
await self.write_entity(entity, file_path)
# Count messages
src/basic_memory/importers/claude_conversations_importer.py
+8 -9
View File
@@ -2,7 +2,6 @@
import logging
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List
from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
@@ -31,7 +30,7 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
"""
try:
# Ensure the destination folder exists
folder_path = await self.ensure_folder_exists(destination_folder)
await self.ensure_folder_exists(destination_folder)
conversations = source_data
@@ -45,15 +44,15 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
# Convert to entity
entity = self._format_chat_content(
base_path=folder_path,
folder=destination_folder,
name=chat_name,
messages=chat["chat_messages"],
created_at=chat["created_at"],
modified_at=chat["updated_at"],
)
# Write file
file_path = self.base_path / Path(f"{entity.frontmatter.metadata['permalink']}.md")
# Write file using relative path - FileService handles base_path
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
await self.write_entity(entity, file_path)
chats_imported += 1
@@ -72,7 +71,7 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
def _format_chat_content(
self,
base_path: Path,
folder: str,
name: str,
messages: List[Dict[str, Any]],
created_at: str,
@@ -81,7 +80,7 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
"""Convert chat messages to Basic Memory entity format.
Args:
base_path: Base path for the entity.
folder: Destination folder name (relative path).
name: Chat name.
messages: List of chat messages.
created_at: Creation timestamp.
@@ -90,10 +89,10 @@ class ClaudeConversationsImporter(Importer[ChatImportResult]):
Returns:
EntityMarkdown instance representing the conversation.
"""
# Generate permalink
# Generate permalink using folder name (relative path)
date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
clean_title = clean_filename(name)
permalink = f"{base_path.name}/{date_prefix}-{clean_title}"
permalink = f"{folder}/{date_prefix}-{clean_title}"
# Format content
content = self._format_chat_markdown(
src/basic_memory/importers/claude_projects_importer.py
+35 -12
View File
@@ -29,9 +29,8 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
"""
try:
# Ensure the base folder exists
base_path = self.base_path
if destination_folder:
base_path = await self.ensure_folder_exists(destination_folder)
await self.ensure_folder_exists(destination_folder)
projects = source_data
@@ -42,20 +41,26 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
for project in projects:
project_dir = clean_filename(project["name"])
# Create project directories using FileService
docs_dir = base_path / project_dir / "docs"
# Create project directories using FileService with relative path
docs_dir = (
f"{destination_folder}/{project_dir}/docs"
if destination_folder
else f"{project_dir}/docs"
)
await self.file_service.ensure_directory(docs_dir)
# Import prompt template if it exists
if prompt_entity := self._format_prompt_markdown(project):
file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
if prompt_entity := self._format_prompt_markdown(project, destination_folder):
# Write file using relative path - FileService handles base_path
file_path = f"{prompt_entity.frontmatter.metadata['permalink']}.md"
await self.write_entity(prompt_entity, file_path)
prompts_imported += 1
# Import project documents
for doc in project.get("docs", []):
entity = self._format_project_markdown(project, doc)
file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
entity = self._format_project_markdown(project, doc, destination_folder)
# Write file using relative path - FileService handles base_path
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
await self.write_entity(entity, file_path)
docs_imported += 1
@@ -71,13 +76,14 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
return self.handle_error("Failed to import Claude projects", e) # pyright: ignore [reportReturnType]
def _format_project_markdown(
self, project: Dict[str, Any], doc: Dict[str, Any]
self, project: Dict[str, Any], doc: Dict[str, Any], destination_folder: str = ""
) -> EntityMarkdown:
"""Format a project document as a Basic Memory entity.
Args:
project: Project data.
doc: Document data.
destination_folder: Optional destination folder prefix.
Returns:
EntityMarkdown instance representing the document.
@@ -90,6 +96,13 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
project_dir = clean_filename(project["name"])
doc_file = clean_filename(doc["filename"])
# Build permalink with optional destination folder prefix
permalink = (
f"{destination_folder}/{project_dir}/docs/{doc_file}"
if destination_folder
else f"{project_dir}/docs/{doc_file}"
)
# Create entity
entity = EntityMarkdown(
frontmatter=EntityFrontmatter(
@@ -98,7 +111,7 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
"title": doc["filename"],
"created": created_at,
"modified": modified_at,
"permalink": f"{project_dir}/docs/{doc_file}",
"permalink": permalink,
"project_name": project["name"],
"project_uuid": project["uuid"],
"doc_uuid": doc["uuid"],
@@ -109,11 +122,14 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
return entity
def _format_prompt_markdown(self, project: Dict[str, Any]) -> Optional[EntityMarkdown]:
def _format_prompt_markdown(
self, project: Dict[str, Any], destination_folder: str = ""
) -> Optional[EntityMarkdown]:
"""Format project prompt template as a Basic Memory entity.
Args:
project: Project data.
destination_folder: Optional destination folder prefix.
Returns:
EntityMarkdown instance representing the prompt template, or None if
@@ -129,6 +145,13 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
# Generate clean project directory name
project_dir = clean_filename(project["name"])
# Build permalink with optional destination folder prefix
permalink = (
f"{destination_folder}/{project_dir}/prompt-template"
if destination_folder
else f"{project_dir}/prompt-template"
)
# Create entity
entity = EntityMarkdown(
frontmatter=EntityFrontmatter(
@@ -137,7 +160,7 @@ class ClaudeProjectsImporter(Importer[ProjectImportResult]):
"title": f"Prompt Template: {project['name']}",
"created": created_at,
"modified": modified_at,
"permalink": f"{project_dir}/prompt-template",
"permalink": permalink,
"project_name": project["name"],
"project_uuid": project["uuid"],
}
src/basic_memory/importers/memory_json_importer.py
+18 -10
View File
@@ -3,7 +3,6 @@
import logging
from typing import Any, Dict, List
from basic_memory.config import get_project_config
from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown, Observation, Relation
from basic_memory.importers.base import Importer
from basic_memory.schemas.importer import EntityImportResult
@@ -27,17 +26,15 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
Returns:
EntityImportResult containing statistics and status of the import.
"""
config = get_project_config()
try:
# First pass - collect all relations by source entity
entity_relations: Dict[str, List[Relation]] = {}
entities: Dict[str, Dict[str, Any]] = {}
skipped_entities: int = 0
# Ensure the base path exists
base_path = config.home # pragma: no cover
# Ensure the destination folder exists if provided
if destination_folder: # pragma: no cover
base_path = await self.ensure_folder_exists(destination_folder)
await self.ensure_folder_exists(destination_folder)
# First pass - collect entities and relations
for line in source_data:
@@ -68,8 +65,19 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
# Get entity type with fallback
entity_type = entity_data.get("entityType") or entity_data.get("type") or "entity"
# Ensure entity type directory exists using FileService
entity_type_dir = base_path / entity_type
# Build permalink with optional destination folder prefix
permalink = (
f"{destination_folder}/{entity_type}/{name}"
if destination_folder
else f"{entity_type}/{name}"
)
# Ensure entity type directory exists using FileService with relative path
entity_type_dir = (
f"{destination_folder}/{entity_type}"
if destination_folder
else entity_type
)
await self.file_service.ensure_directory(entity_type_dir)
# Get observations with fallback to empty list
@@ -80,7 +88,7 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
metadata={
"type": entity_type,
"title": name,
"permalink": f"{entity_type}/{name}",
"permalink": permalink,
}
),
content=f"# {name}\n",
@@ -88,8 +96,8 @@ class MemoryJsonImporter(Importer[EntityImportResult]):
relations=entity_relations.get(name, []),
)
# Write entity file
file_path = base_path / f"{entity_type}/{name}.md"
# Write file using relative path - FileService handles base_path
file_path = f"{entity.frontmatter.metadata['permalink']}.md"
await self.write_entity(entity, file_path)
entities_created += 1
tests/importers/test_importer_base.py
+10 -14
View File
@@ -69,15 +69,15 @@ def test_importer(tmp_path, mock_markdown_processor, mock_file_service):
@pytest.mark.asyncio
async def test_import_data_success(test_importer, mock_file_service, tmp_path):
async def test_import_data_success(test_importer, mock_file_service):
"""Test successful import_data implementation."""
result = await test_importer.import_data({}, "test_folder")
assert result.success
assert result.import_count == {"files": 1}
assert result.error_message is None
# Verify file_service.ensure_directory was called
mock_file_service.ensure_directory.assert_called_once_with(tmp_path / "test_folder")
# Verify file_service.ensure_directory was called with relative path
mock_file_service.ensure_directory.assert_called_once_with("test_folder")
@pytest.mark.asyncio
@@ -105,19 +105,15 @@ async def test_write_entity(test_importer, mock_markdown_processor, mock_file_se
@pytest.mark.asyncio
async def test_ensure_folder_exists(test_importer, mock_file_service, tmp_path):
async def test_ensure_folder_exists(test_importer, mock_file_service):
"""Test ensure_folder_exists method."""
# Test with simple folder
folder_path = await test_importer.ensure_folder_exists("test_folder")
assert folder_path == tmp_path / "test_folder"
# Test with simple folder - now passes relative path to FileService
await test_importer.ensure_folder_exists("test_folder")
mock_file_service.ensure_directory.assert_called_with("test_folder")
# Verify file_service.ensure_directory was called
mock_file_service.ensure_directory.assert_called_with(tmp_path / "test_folder")
# Test with nested folder
nested_path = await test_importer.ensure_folder_exists("nested/folder/path")
assert nested_path == tmp_path / "nested" / "folder" / "path"
mock_file_service.ensure_directory.assert_called_with(tmp_path / "nested" / "folder" / "path")
# Test with nested folder - FileService handles base_path resolution
await test_importer.ensure_folder_exists("nested/folder/path")
mock_file_service.ensure_directory.assert_called_with("nested/folder/path")
@pytest.mark.asyncio