mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
fix: add logfire spans to cli
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
"""Database management commands."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
@@ -13,13 +15,14 @@ def reset(
|
||||
reindex: bool = typer.Option(False, "--reindex", help="Rebuild indices from filesystem"),
|
||||
): # pragma: no cover
|
||||
"""Reset database (drop all tables and recreate)."""
|
||||
if typer.confirm("This will delete all data. Are you sure?"):
|
||||
logger.info("Resetting database...")
|
||||
asyncio.run(migrations.reset_database())
|
||||
with logfire.span("reset"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
if typer.confirm("This will delete all data in your db. Are you sure?"):
|
||||
logger.info("Resetting database...")
|
||||
asyncio.run(migrations.reset_database())
|
||||
|
||||
if reindex:
|
||||
# Import and run sync
|
||||
from basic_memory.cli.commands.sync import sync
|
||||
if reindex:
|
||||
# Import and run sync
|
||||
from basic_memory.cli.commands.sync import sync
|
||||
|
||||
logger.info("Rebuilding search index from filesystem...")
|
||||
sync(watch=False) # pyright: ignore
|
||||
logger.info("Rebuilding search index from filesystem...")
|
||||
sync(watch=False) # pyright: ignore
|
||||
|
||||
@@ -6,6 +6,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated, Set, Optional
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -225,35 +226,38 @@ def import_chatgpt(
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if conversations_json:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
with logfire.span("import chatgpt"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
try:
|
||||
if conversations_json:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_chatgpt_json(conversations_json, folder, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(
|
||||
f"\nImporting chats from {conversations_json}...writing to {base_path}"
|
||||
)
|
||||
results = asyncio.run(
|
||||
process_chatgpt_json(conversations_json, folder, markdown_processor)
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -6,6 +6,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -178,34 +179,35 @@ def import_claude(
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
with logfire.span("import claude conversations"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
try:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_conversations_json(conversations_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_conversations_json(conversations_json, base_path, markdown_processor)
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Annotated, Optional
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -160,36 +161,36 @@ def import_projects(
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
with logfire.span("import claude projects"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
try:
|
||||
if projects_json:
|
||||
if not projects_json.exists():
|
||||
typer.echo(f"Error: File not found: {projects_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
if projects_json:
|
||||
if not projects_json.exists():
|
||||
typer.echo(f"Error: File not found: {projects_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_projects_json(projects_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['documents']} project documents\n"
|
||||
f"Imported {results['prompts']} prompt templates",
|
||||
expand=False,
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_projects_json(projects_json, base_path, markdown_processor)
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['documents']} project documents\n"
|
||||
f"Imported {results['prompts']} prompt templates",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -113,32 +114,33 @@ def memory_json(
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
if not json_path.exists():
|
||||
typer.echo(f"Error: File not found: {json_path}", err=True)
|
||||
raise typer.Exit(1)
|
||||
with logfire.span("import memory_json"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
if not json_path.exists():
|
||||
typer.echo(f"Error: File not found: {json_path}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
try:
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
try:
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home
|
||||
console.print(f"\nImporting from {json_path}...writing to {base_path}")
|
||||
results = asyncio.run(process_memory_json(json_path, base_path, markdown_processor))
|
||||
# Process the file
|
||||
base_path = config.home
|
||||
console.print(f"\nImporting from {json_path}...writing to {base_path}")
|
||||
results = asyncio.run(process_memory_json(json_path, base_path, markdown_processor))
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Created {results['entities']} entities\n"
|
||||
f"Added {results['relations']} relations",
|
||||
expand=False,
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Created {results['entities']} entities\n"
|
||||
f"Added {results['relations']} relations",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -17,4 +17,4 @@ def mcp(): # pragma: no cover
|
||||
home_dir = config.home
|
||||
logger.info(f"Starting Basic Memory MCP server {basic_memory.__version__}")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
mcp_server.run()
|
||||
mcp_server.run()
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
from typing import Set, Dict
|
||||
|
||||
import logfire
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
@@ -146,9 +147,10 @@ def status(
|
||||
verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed file information"),
|
||||
):
|
||||
"""Show sync status between files and database."""
|
||||
try:
|
||||
sync_service = asyncio.run(get_file_change_scanner())
|
||||
asyncio.run(run_status(sync_service, verbose)) # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.exception(f"Error checking status: {e}")
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
with logfire.span("status"): # pyright: ignore [reportGeneralTypeIssues]
|
||||
try:
|
||||
sync_service = asyncio.run(get_file_change_scanner())
|
||||
asyncio.run(run_status(sync_service, verbose)) # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.exception(f"Error checking status: {e}")
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
|
||||
@@ -4,7 +4,6 @@ from enum import Enum, auto
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator, Optional
|
||||
|
||||
import logfire
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from alembic import command
|
||||
|
||||
@@ -4,7 +4,6 @@ import logfire
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_get, call_post
|
||||
from basic_memory.schemas.base import Permalink
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
from basic_memory.schemas.request import (
|
||||
GetEntitiesRequest,
|
||||
@@ -25,7 +24,7 @@ async def get_entity(identifier: str) -> EntityResponse:
|
||||
Args:
|
||||
identifier: Path identifier for the entity
|
||||
"""
|
||||
with logfire.span("Getting entity", permalink=identifier) as s:
|
||||
with logfire.span("Getting entity", permalink=identifier): # pyright: ignore [reportGeneralTypeIssues]
|
||||
permalink = memory_url_path(identifier)
|
||||
url = f"/knowledge/entities/{permalink}"
|
||||
response = await call_get(client, url)
|
||||
@@ -44,10 +43,14 @@ async def get_entities(request: GetEntitiesRequest) -> EntityListResponse:
|
||||
Returns:
|
||||
EntityListResponse containing complete details for each requested entity
|
||||
"""
|
||||
with logfire.span("Getting multiple entities", permalink_count=len(request.permalinks)) as s:
|
||||
with logfire.span("Getting multiple entities", permalink_count=len(request.permalinks)): # pyright: ignore [reportGeneralTypeIssues]
|
||||
url = "/knowledge/entities"
|
||||
response = await call_get(
|
||||
client, url, params=[("permalink", memory_url_path(identifier)) for identifier in request.permalinks]
|
||||
client,
|
||||
url,
|
||||
params=[
|
||||
("permalink", memory_url_path(identifier)) for identifier in request.permalinks
|
||||
],
|
||||
)
|
||||
return EntityListResponse.model_validate(response.json())
|
||||
|
||||
@@ -57,9 +60,9 @@ async def get_entities(request: GetEntitiesRequest) -> EntityListResponse:
|
||||
)
|
||||
async def delete_entities(request: DeleteEntitiesRequest) -> DeleteEntitiesResponse:
|
||||
"""Delete entities from the knowledge graph."""
|
||||
with logfire.span("Deleting entities", permalink_count=len(request.permalinks)) as s:
|
||||
with logfire.span("Deleting entities", permalink_count=len(request.permalinks)): # pyright: ignore [reportGeneralTypeIssues]
|
||||
url = "/knowledge/entities/delete"
|
||||
|
||||
|
||||
request.permalinks = [memory_url_path(permlink) for permlink in request.permalinks]
|
||||
response = await call_post(client, url, json=request.model_dump())
|
||||
return DeleteEntitiesResponse.model_validate(response.json())
|
||||
return DeleteEntitiesResponse.model_validate(response.json())
|
||||
|
||||
@@ -66,7 +66,7 @@ async def build_context(
|
||||
# Research the history of a feature
|
||||
build_context("memory://features/knowledge-graph", timeframe="3 months ago")
|
||||
"""
|
||||
with logfire.span("Building context", url=url, depth=depth, timeframe=timeframe) as s:
|
||||
with logfire.span("Building context", url=url, depth=depth, timeframe=timeframe): # pyright: ignore [reportGeneralTypeIssues]
|
||||
logger.info(f"Building context from {url}")
|
||||
url = normalize_memory_url(url)
|
||||
response = await call_get(
|
||||
@@ -134,7 +134,7 @@ async def recent_activity(
|
||||
- For focused queries, consider using build_context with a specific URI
|
||||
- Max timeframe is 1 year in the past
|
||||
"""
|
||||
with logfire.span("Getting recent activity", type=type, depth=depth, timeframe=timeframe) as s:
|
||||
with logfire.span("Getting recent activity", type=type, depth=depth, timeframe=timeframe): # pyright: ignore [reportGeneralTypeIssues]
|
||||
logger.info(
|
||||
f"Getting recent activity from {type}, depth={depth}, timeframe={timeframe}, max_results={max_results}"
|
||||
)
|
||||
@@ -151,4 +151,4 @@ async def recent_activity(
|
||||
"/memory/recent",
|
||||
params=params,
|
||||
)
|
||||
return GraphContext.model_validate(response.json())
|
||||
return GraphContext.model_validate(response.json())
|
||||
|
||||
@@ -62,7 +62,7 @@ async def write_note(
|
||||
- Relation counts (resolved/unresolved)
|
||||
- Tags if present
|
||||
"""
|
||||
with logfire.span("Writing note", title=title, folder=folder) as s:
|
||||
with logfire.span("Writing note", title=title, folder=folder): # pyright: ignore [reportGeneralTypeIssues]
|
||||
logger.info(f"Writing note folder:'{folder}' title: '{title}'")
|
||||
|
||||
# Create the entity request
|
||||
@@ -81,25 +81,29 @@ async def write_note(
|
||||
url = f"/knowledge/entities/{entity.permalink}"
|
||||
response = await call_put(client, url, json=entity.model_dump())
|
||||
result = EntityResponse.model_validate(response.json())
|
||||
|
||||
|
||||
# Format semantic summary based on status code
|
||||
action = "Created" if response.status_code == 201 else "Updated"
|
||||
summary = [f"# {action} {result.file_path} ({result.checksum[:8]})", f"permalink: {result.permalink}"]
|
||||
assert result.checksum is not None
|
||||
summary = [
|
||||
f"# {action} {result.file_path} ({result.checksum[:8]})",
|
||||
f"permalink: {result.permalink}",
|
||||
]
|
||||
|
||||
if result.observations:
|
||||
categories = {}
|
||||
for obs in result.observations:
|
||||
categories[obs.category] = categories.get(obs.category, 0) + 1
|
||||
|
||||
|
||||
summary.append("\n## Observations")
|
||||
for category, count in sorted(categories.items()):
|
||||
summary.append(f"- {category}: {count}")
|
||||
|
||||
|
||||
if result.relations:
|
||||
unresolved = sum(1 for r in result.relations if not r.to_id)
|
||||
resolved = len(result.relations) - unresolved
|
||||
|
||||
summary.append("\n## Relations")
|
||||
|
||||
summary.append("\n## Relations")
|
||||
summary.append(f"- Resolved: {resolved}")
|
||||
if unresolved:
|
||||
summary.append(f"- Unresolved: {unresolved}")
|
||||
@@ -164,7 +168,7 @@ async def read_note(identifier: str) -> str:
|
||||
- Last modified timestamp
|
||||
- Content checksum
|
||||
"""
|
||||
with logfire.span("Reading note", identifier=identifier) as s:
|
||||
with logfire.span("Reading note", identifier=identifier): # pyright: ignore [reportGeneralTypeIssues]
|
||||
logger.info(f"Reading note {identifier}")
|
||||
url = memory_url_path(identifier)
|
||||
response = await call_get(client, f"/resource/{url}")
|
||||
@@ -188,7 +192,7 @@ async def delete_note(identifier: str) -> bool:
|
||||
# Delete by permalink
|
||||
delete_note("notes/project-planning")
|
||||
"""
|
||||
with logfire.span("Deleting note", identifier=identifier) as s:
|
||||
with logfire.span("Deleting note", identifier=identifier): # pyright: ignore [reportGeneralTypeIssues]
|
||||
response = await call_delete(client, f"/knowledge/entities/{identifier}")
|
||||
result = DeleteEntitiesResponse.model_validate(response.json())
|
||||
return result.deleted
|
||||
return result.deleted
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Search tools for Basic Memory MCP server."""
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
|
||||
@@ -24,7 +25,7 @@ async def search(query: SearchQuery) -> SearchResponse:
|
||||
Returns:
|
||||
SearchResponse with search results and metadata
|
||||
"""
|
||||
with logfire.span("Searching for {query}", qurey=query):
|
||||
with logfire.span("Searching for {query}", qurey=query): # pyright: ignore [reportGeneralTypeIssues]
|
||||
logger.info(f"Searching for {query}")
|
||||
response = await call_post(client, "/search/", json=query.model_dump())
|
||||
return SearchResponse.model_validate(response.json())
|
||||
|
||||
@@ -34,4 +34,4 @@ class DeleteEntitiesRequest(BaseModel):
|
||||
4. Deletes the corresponding markdown file
|
||||
"""
|
||||
|
||||
permalinks: Annotated[List[Permalink], MinLen(1)]
|
||||
permalinks: Annotated[List[Permalink], MinLen(1)]
|
||||
|
||||
@@ -55,4 +55,4 @@ class GetEntitiesRequest(BaseModel):
|
||||
|
||||
|
||||
class CreateRelationsRequest(BaseModel):
|
||||
relations: List[Relation]
|
||||
relations: List[Relation]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
@@ -61,105 +62,113 @@ class SyncService:
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""Sync knowledge files with database."""
|
||||
changes = await self.scanner.find_knowledge_changes(directory)
|
||||
logger.info(f"Found {changes.total_changes} knowledge changes")
|
||||
|
||||
# Handle moves first
|
||||
for old_path, new_path in changes.moves.items():
|
||||
logger.debug(f"Moving entity: {old_path} -> {new_path}")
|
||||
entity = await self.entity_repository.get_by_file_path(old_path)
|
||||
if entity:
|
||||
# Update file_path but keep the same permalink for link stability
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
|
||||
)
|
||||
# update search index
|
||||
if updated:
|
||||
await self.search_service.index_entity(updated)
|
||||
with logfire.span("sync", directory=directory): # pyright: ignore [reportGeneralTypeIssues]
|
||||
changes = await self.scanner.find_knowledge_changes(directory)
|
||||
logger.info(f"Found {changes.total_changes} knowledge changes")
|
||||
|
||||
# Handle deletions next
|
||||
# remove rows from db for files no longer present
|
||||
for path in changes.deleted:
|
||||
await self.handle_entity_deletion(path)
|
||||
# Handle moves first
|
||||
for old_path, new_path in changes.moves.items():
|
||||
logger.debug(f"Moving entity: {old_path} -> {new_path}")
|
||||
entity = await self.entity_repository.get_by_file_path(old_path)
|
||||
if entity:
|
||||
# Update file_path but keep the same permalink for link stability
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
|
||||
)
|
||||
# update search index
|
||||
if updated:
|
||||
await self.search_service.index_entity(updated)
|
||||
|
||||
# Parse files that need updating
|
||||
parsed_entities: Dict[str, EntityMarkdown] = {}
|
||||
# Handle deletions next
|
||||
# remove rows from db for files no longer present
|
||||
for path in changes.deleted:
|
||||
await self.handle_entity_deletion(path)
|
||||
|
||||
for path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / path)
|
||||
parsed_entities[path] = entity_markdown
|
||||
# Parse files that need updating
|
||||
parsed_entities: Dict[str, EntityMarkdown] = {}
|
||||
|
||||
# First pass: Create/update entities
|
||||
# entities will have a null checksum to indicate they are not complete
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
# Get unique permalink and update markdown if needed
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
Path(path), markdown=entity_markdown
|
||||
)
|
||||
for path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / path)
|
||||
parsed_entities[path] = entity_markdown
|
||||
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
# Add/update permalink in frontmatter
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {path}")
|
||||
|
||||
# update markdown
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
|
||||
# update file frontmatter
|
||||
updated_checksum = await file_utils.update_frontmatter(
|
||||
directory / path, {"permalink": permalink}
|
||||
# First pass: Create/update entities
|
||||
# entities will have a null checksum to indicate they are not complete
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
# Get unique permalink and update markdown if needed
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
Path(path), markdown=entity_markdown
|
||||
)
|
||||
|
||||
# Update checksum in changes report since file was modified
|
||||
changes.checksums[path] = updated_checksum
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
# Add/update permalink in frontmatter
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {path}")
|
||||
|
||||
# if the file is new, create an entity
|
||||
if path in changes.new:
|
||||
# Create entity with final permalink
|
||||
logger.debug(f"Creating new entity_markdown: {path}")
|
||||
await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
|
||||
# otherwise we need to update the entity and observations
|
||||
else:
|
||||
logger.debug(f"Updating entity_markdown: {path}")
|
||||
await self.entity_service.update_entity_and_observations(
|
||||
# update markdown
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
|
||||
# update file frontmatter
|
||||
updated_checksum = await file_utils.update_frontmatter(
|
||||
directory / path, {"permalink": permalink}
|
||||
)
|
||||
|
||||
# Update checksum in changes report since file was modified
|
||||
changes.checksums[path] = updated_checksum
|
||||
|
||||
# if the file is new, create an entity
|
||||
if path in changes.new:
|
||||
# Create entity with final permalink
|
||||
logger.debug(f"Creating new entity_markdown: {path}")
|
||||
await self.entity_service.create_entity_from_markdown(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
# otherwise we need to update the entity and observations
|
||||
else:
|
||||
logger.debug(f"Updating entity_markdown: {path}")
|
||||
await self.entity_service.update_entity_and_observations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# Second pass
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {path}")
|
||||
|
||||
# Process relations
|
||||
checksum = changes.checksums[path]
|
||||
entity = await self.entity_service.update_entity_relations(
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# Second pass
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {path}")
|
||||
# add to search index
|
||||
await self.search_service.index_entity(entity)
|
||||
|
||||
# Process relations
|
||||
checksum = changes.checksums[path]
|
||||
entity = await self.entity_service.update_entity_relations(Path(path), entity_markdown)
|
||||
# Set final checksum to mark sync complete
|
||||
await self.entity_repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
# add to search index
|
||||
await self.search_service.index_entity(entity)
|
||||
|
||||
# Set final checksum to mark sync complete
|
||||
await self.entity_repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
# Third pass: Try to resolve any forward references
|
||||
logger.debug("Attempting to resolve forward references")
|
||||
for relation in await self.relation_repository.find_unresolved_relations():
|
||||
target_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
|
||||
# check we found a link that is not the source
|
||||
if target_entity and target_entity.id != relation.from_id:
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
|
||||
# Third pass: Try to resolve any forward references
|
||||
logger.debug("Attempting to resolve forward references")
|
||||
for relation in await self.relation_repository.find_unresolved_relations():
|
||||
target_entity = await self.entity_service.link_resolver.resolve_link(
|
||||
relation.to_name
|
||||
)
|
||||
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title, # Update to actual title
|
||||
},
|
||||
# check we found a link that is not the source
|
||||
if target_entity and target_entity.id != relation.from_id:
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
|
||||
# update search index
|
||||
await self.search_service.index_entity(target_entity)
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title, # Update to actual title
|
||||
},
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
|
||||
return changes
|
||||
# update search index
|
||||
await self.search_service.index_entity(target_entity)
|
||||
|
||||
return changes
|
||||
|
||||
@@ -32,6 +32,7 @@ async def test_get_single_entity(client):
|
||||
assert entity.permalink == "test/test-note"
|
||||
assert len(entity.observations) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_single_entity_memory_url(client):
|
||||
"""Test retrieving a single entity."""
|
||||
@@ -81,6 +82,7 @@ async def test_get_multiple_entities(client):
|
||||
assert "test/test-note-1" in permalinks
|
||||
assert "test/test-note-2" in permalinks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_multiple_entities_memory_ur(client):
|
||||
"""Test retrieving multiple entities."""
|
||||
@@ -97,7 +99,9 @@ async def test_get_multiple_entities_memory_ur(client):
|
||||
)
|
||||
|
||||
# Get both entities
|
||||
request = GetEntitiesRequest(permalinks=["memory://test/test-note-1", "memory://test/test-note-2"])
|
||||
request = GetEntitiesRequest(
|
||||
permalinks=["memory://test/test-note-1", "memory://test/test-note-2"]
|
||||
)
|
||||
response = await get_entities(request)
|
||||
|
||||
# Verify we got both entities
|
||||
@@ -128,6 +132,7 @@ async def test_delete_entities(client):
|
||||
with pytest.raises(ToolError):
|
||||
await get_entity("test/test-note")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_entities_memory_url(client):
|
||||
"""Test deleting entities."""
|
||||
|
||||
Reference in New Issue
Block a user