mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
feat: Import claude.ai data (#8)
Import Claude.ai conversation and project data to basic-memory Markdown format. --------- Co-authored-by: phernandez <phernandez@basicmachines.co>
This commit is contained in:
Paul Hernandez
@@ -10,4 +10,11 @@ setup_logging(log_file=".basic-memory/basic-memory-cli.log") # pragma: no cover
|
||||
|
||||
asyncio.run(db.run_migrations(config))
|
||||
|
||||
app = typer.Typer()
|
||||
app = typer.Typer(name="basic-memory")
|
||||
|
||||
import_app = typer.Typer()
|
||||
app.add_typer(import_app, name="import")
|
||||
|
||||
|
||||
claude_app = typer.Typer()
|
||||
import_app.add_typer(claude_app, name="claude")
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Import command for basic-memory CLI to import chat data from conversations2.json format."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
# Remove invalid characters and convert spaces
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_timestamp(ts: str) -> str:
|
||||
"""Format ISO timestamp for display."""
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def format_chat_markdown(
|
||||
name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
|
||||
) -> str:
|
||||
"""Format chat as clean markdown."""
|
||||
|
||||
# Start with frontmatter and title
|
||||
lines = [
|
||||
f"# {name}\n",
|
||||
]
|
||||
|
||||
# Add messages
|
||||
for msg in messages:
|
||||
# Format timestamp
|
||||
ts = format_timestamp(msg["created_at"])
|
||||
|
||||
# Add message header
|
||||
lines.append(f"### {msg['sender'].title()} ({ts})")
|
||||
|
||||
# Handle message content
|
||||
content = msg.get("text", "")
|
||||
if msg.get("content"):
|
||||
content = " ".join(c.get("text", "") for c in msg["content"])
|
||||
lines.append(content)
|
||||
|
||||
# Handle attachments
|
||||
attachments = msg.get("attachments", [])
|
||||
for attachment in attachments:
|
||||
if "file_name" in attachment:
|
||||
lines.append(f"\n**Attachment: {attachment['file_name']}**")
|
||||
if "extracted_content" in attachment:
|
||||
lines.append("```")
|
||||
lines.append(attachment["extracted_content"])
|
||||
lines.append("```")
|
||||
|
||||
# Add spacing between messages
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_chat_content(
|
||||
base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
|
||||
) -> EntityMarkdown:
|
||||
"""Convert chat messages to Basic Memory entity format."""
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(name)
|
||||
permalink = f"{base_path}/{date_prefix}-{clean_title}"
|
||||
|
||||
# Format content
|
||||
content = format_chat_markdown(
|
||||
name=name,
|
||||
messages=messages,
|
||||
created_at=created_at,
|
||||
modified_at=modified_at,
|
||||
permalink=permalink,
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "conversation",
|
||||
"title": name,
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": permalink,
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_conversations_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import chat data from conversations2.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading chat data...", total=None)
|
||||
|
||||
# Read chat data - handle array of arrays format
|
||||
data = json.loads(json_path.read_text())
|
||||
conversations = [chat for chat in data]
|
||||
progress.update(read_task, total=len(conversations))
|
||||
|
||||
# Process each conversation
|
||||
messages_imported = 0
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
# Convert to entity
|
||||
entity = format_chat_content(
|
||||
base_path=base_path,
|
||||
name=chat["name"],
|
||||
messages=chat["chat_messages"],
|
||||
created_at=chat["created_at"],
|
||||
modified_at=chat["updated_at"],
|
||||
)
|
||||
|
||||
# Write file
|
||||
file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
|
||||
chats_imported += 1
|
||||
messages_imported += len(chat["chat_messages"])
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"conversations": chats_imported, "messages": messages_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.")
|
||||
def import_claude(
|
||||
conversations_json: Annotated[
|
||||
Path, typer.Argument(..., help="Path to conversations.json file")
|
||||
] = Path("conversations.json"),
|
||||
folder: Annotated[
|
||||
str, typer.Option(help="The folder to place the files in.")
|
||||
] = "conversations",
|
||||
):
|
||||
"""Import chat conversations from conversations2.json format.
|
||||
|
||||
This command will:
|
||||
1. Read chat data and nested messages
|
||||
2. Create markdown files for each conversation
|
||||
3. Format content in clean, readable markdown
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_conversations_json(conversations_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Import command for basic-memory CLI to import project data from Claude.ai."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Annotated, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown:
|
||||
"""Format a project document as a Basic Memory entity."""
|
||||
|
||||
# Extract timestamps
|
||||
created_at = doc.get("created_at") or project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean names for organization
|
||||
project_dir = clean_filename(project["name"])
|
||||
doc_file = clean_filename(doc["filename"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "project_doc",
|
||||
"title": doc["filename"],
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/docs/{doc_file}",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
"doc_uuid": doc["uuid"],
|
||||
}
|
||||
),
|
||||
content=doc["content"],
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]:
|
||||
"""Format project prompt template as a Basic Memory entity."""
|
||||
|
||||
if not project.get("prompt_template"):
|
||||
return None
|
||||
|
||||
# Extract timestamps
|
||||
created_at = project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean project directory name
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "prompt_template",
|
||||
"title": f"Prompt Template: {project['name']}",
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/prompt-template",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
}
|
||||
),
|
||||
content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_projects_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import project data from Claude.ai projects.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading project data...", total=None)
|
||||
|
||||
# Read project data
|
||||
data = json.loads(json_path.read_text())
|
||||
progress.update(read_task, total=len(data))
|
||||
|
||||
# Track import counts
|
||||
docs_imported = 0
|
||||
prompts_imported = 0
|
||||
|
||||
# Process each project
|
||||
for project in data:
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create project directories
|
||||
docs_dir = base_path / project_dir / "docs"
|
||||
docs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Import prompt template if it exists
|
||||
if prompt_entity := format_prompt_markdown(project):
|
||||
file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, prompt_entity)
|
||||
prompts_imported += 1
|
||||
|
||||
# Import project documents
|
||||
for doc in project.get("docs", []):
|
||||
entity = format_project_markdown(project, doc)
|
||||
file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
docs_imported += 1
|
||||
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"documents": docs_imported, "prompts": prompts_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="projects", help="Import projects from Claude.ai.")
|
||||
def import_projects(
|
||||
projects_json: Annotated[Path, typer.Argument(..., help="Path to projects.json file")] = Path(
|
||||
"projects.json"
|
||||
),
|
||||
base_folder: Annotated[
|
||||
str, typer.Option(help="The base folder to place project files in.")
|
||||
] = "projects",
|
||||
):
|
||||
"""Import project data from Claude.ai.
|
||||
|
||||
This command will:
|
||||
1. Create a directory for each project
|
||||
2. Store docs in a docs/ subdirectory
|
||||
3. Place prompt template in project root
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if projects_json:
|
||||
if not projects_json.exists():
|
||||
typer.echo(f"Error: File not found: {projects_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_projects_json(projects_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['documents']} project documents\n"
|
||||
f"Imported {results['prompts']} prompt templates",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
@@ -11,7 +11,7 @@ from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation
|
||||
@@ -98,9 +98,11 @@ async def get_markdown_processor() -> MarkdownProcessor:
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@app.command()
|
||||
def import_json(
|
||||
json_path: Path = typer.Argument(..., help="Path to memory.json file to import"),
|
||||
@import_app.command()
|
||||
def memory_json(
|
||||
json_path: Annotated[Path, typer.Argument(..., help="Path to memory.json file")] = Path(
|
||||
"memory.json"
|
||||
),
|
||||
):
|
||||
"""Import entities and relations from a memory.json file.
|
||||
|
||||
|
||||
@@ -4,7 +4,15 @@ from basic_memory.cli.app import app # pragma: no cover
|
||||
from basic_memory.utils import setup_logging # pragma: no cover
|
||||
|
||||
# Register commands
|
||||
from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp # noqa: F401 # pragma: no cover
|
||||
from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
status,
|
||||
sync,
|
||||
db,
|
||||
import_memory_json,
|
||||
mcp,
|
||||
import_claude_conversations,
|
||||
import_claude_projects,
|
||||
)
|
||||
|
||||
|
||||
# Set up logging when module is imported
|
||||
|
||||
Reference in New Issue
Block a user