admin/basicmachines-co-basic-memory
1
0
Fork 0
mirror of https://github.com/basicmachines-co/basic-memory synced 2026-06-21 13:47:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

change entity.path_id to entity.permalink

Browse Source
This commit is contained in:
phernandez
2025-01-12 16:47:21 -06:00
parent 1f374e0133
commit c5d21276de
52 changed files with 714 additions and 687 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic_memory/api/routers/knowledge_router.py
+25 -21
View File
@@ -5,7 +5,10 @@ from loguru import logger
from basic_memory.deps import (
EntityServiceDep,
get_search_service, RelationServiceDep, ObservationServiceDep, FileServiceDep,
get_search_service,
RelationServiceDep,
ObservationServiceDep,
FileServiceDep,
)
from basic_memory.schemas import (
CreateEntityRequest,
@@ -47,9 +50,9 @@ async def create_entities(
)
@router.put("/entities/{path_id:path}", response_model=EntityResponse)
@router.put("/entities/{permalink:path}", response_model=EntityResponse)
async def update_entity(
path_id: PathId,
permalink: PathId,
data: UpdateEntityRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
@@ -61,7 +64,7 @@ async def update_entity(
update_data = data.model_dump(exclude_none=True)
# Update the entity
updated_entity = await entity_service.update_entity(path_id, **update_data)
updated_entity = await entity_service.update_entity(permalink, **update_data)
# Reindex since content changed
await search_service.index_entity(updated_entity, background_tasks=background_tasks)
@@ -69,7 +72,7 @@ async def update_entity(
return EntityResponse.model_validate(updated_entity)
except EntityNotFoundError:
raise HTTPException(status_code=404, detail=f"Entity with {path_id} not found")
raise HTTPException(status_code=404, detail=f"Entity with {permalink} not found")
@router.post("/relations", response_model=EntityListResponse)
@@ -99,9 +102,9 @@ async def add_observations(
search_service=Depends(get_search_service),
) -> EntityResponse:
"""Add observations to an entity and update search index."""
logger.debug(f"Adding observations to entity: {data.path_id}")
logger.debug(f"Adding observations to entity: {data.permalink}")
updated_entity = await observation_service.add_observations(
data.path_id, data.observations, data.context
data.permalink, data.observations, data.context
)
# Reindex the entity with new observations
@@ -113,22 +116,22 @@ async def add_observations(
## Read endpoints
@router.get("/entities/{path_id:path}", response_model=EntityResponse)
@router.get("/entities/{permalink:path}", response_model=EntityResponse)
async def get_entity(
entity_service: EntityServiceDep,
file_service: FileServiceDep,
path_id: PathId,
content: bool = False, # New parameter
entity_service: EntityServiceDep,
file_service: FileServiceDep,
permalink: PathId,
content: bool = False, # New parameter
) -> EntityResponse:
"""Get a specific entity by ID.
Args:
path_id: Entity path ID
permalink: Entity path ID
content: If True, include full file content
:param entity_service: EntityService
"""
try:
entity = await entity_service.get_by_path_id(path_id)
entity = await entity_service.get_by_permalink(permalink)
entity_response = EntityResponse.model_validate(entity)
if content: # Load content if requested
@@ -137,14 +140,15 @@ async def get_entity(
return entity_response
except EntityNotFoundError:
raise HTTPException(status_code=404, detail=f"Entity with {path_id} not found")
raise HTTPException(status_code=404, detail=f"Entity with {permalink} not found")
@router.post("/nodes", response_model=EntityListResponse)
async def open_nodes(
data: OpenNodesRequest, entity_service: EntityServiceDep
) -> EntityListResponse:
"""Open specific nodes"""
entities = await entity_service.open_nodes(data.path_ids)
entities = await entity_service.open_nodes(data.permalinks)
return EntityListResponse(
entities=[EntityResponse.model_validate(entity) for entity in entities]
)
@@ -161,11 +165,11 @@ async def delete_entities(
search_service=Depends(get_search_service),
) -> DeleteEntitiesResponse:
"""Delete entities and remove from search index."""
deleted = await entity_service.delete_entities(data.path_ids)
deleted = await entity_service.delete_entities(data.permalinks)
# Remove each deleted entity from search index
for path_id in data.path_ids:
background_tasks.add_task(search_service.delete_by_path_id, path_id)
for permalink in data.permalinks:
background_tasks.add_task(search_service.delete_by_permalink, permalink)
return DeleteEntitiesResponse(deleted=deleted)
@@ -178,8 +182,8 @@ async def delete_observations(
search_service=Depends(get_search_service),
) -> EntityResponse:
"""Delete observations and update search index."""
path_id = data.path_id
updated_entity = await observation_service.delete_observations(path_id, data.observations)
permalink = data.permalink
updated_entity = await observation_service.delete_observations(permalink, data.observations)
# Reindex the entity since observations changed
await search_service.index_entity(updated_entity, background_tasks=background_tasks)
src/basic_memory/markdown/entity_parser.py
+6 -8
View File
@@ -24,14 +24,12 @@ class EntityParser:
"""Parser for markdown files into Entity objects."""
def __init__(self, base_path: Path):
"""Initialize parser with base path for relative path_id generation."""
"""Initialize parser with base path for relative permalink generation."""
self.base_path = base_path.resolve()
self.md = (MarkdownIt()
.use(observation_plugin)
.use(relation_plugin))
self.md = MarkdownIt().use(observation_plugin).use(relation_plugin)
def get_path_id(self, file_path: Path) -> str:
"""Get path_id from file path relative to base_path.
def get_permalink(self, file_path: Path) -> str:
"""Get permalink from file path relative to base_path.
Example:
base_path: /project/root
@@ -61,13 +59,13 @@ class EntityParser:
post = frontmatter.load(str(file_path))
# Extract or generate required fields
path_id = post.metadata.get("id") or self.get_path_id(file_path)
permalink = post.metadata.get("id") or self.get_permalink(file_path)
stats = file_path.stat()
# Parse frontmatter
entity_frontmatter = EntityFrontmatter(
type=str(post.metadata.get("type", "note")),
id=path_id,
id=permalink,
title=str(post.metadata.get("title", file_path.name)),
created=self.parse_date(post.metadata.get("created"))
or datetime.fromtimestamp(stats.st_ctime),
src/basic_memory/markdown/knowledge_writer.py
+1 -1
View File
@@ -16,7 +16,7 @@ class KnowledgeWriter:
async def format_frontmatter(self, entity: EntityModel) -> dict:
"""Generate frontmatter metadata for entity."""
frontmatter = {
"id": entity.path_id,
"id": entity.permalink,
"type": entity.entity_type,
"created": entity.created_at.isoformat(),
"modified": entity.updated_at.isoformat(),
src/basic_memory/mcp/tools/activity.py
+16 -16
View File
@@ -30,7 +30,7 @@ print(f"Relations changed: {activity.summary.relation_changes}")
print("\\nMost active paths:")
for path in activity.summary.most_active_paths:
print(f"- {path}")
"""
""",
},
{
"name": "Document Changes",
@@ -44,9 +44,9 @@ docs = await get_recent_activity(
# Show document evolution chronologically
for change in sorted(docs.changes, key=lambda x: x.timestamp):
print(f"{change.timestamp}: {change.path_id}")
print(f"{change.timestamp}: {change.permalink}")
print(f" {change.change_type}: {change.summary}")
"""
""",
},
{
"name": "Knowledge Evolution",
@@ -69,16 +69,16 @@ for type_, changes in changes_by_type.items():
entity_changes = defaultdict(int)
for change in weekly.changes:
if change.activity_type == "entity":
entity_changes[change.path_id] += 1
entity_changes[change.permalink] += 1
print("\\nMost active entities:")
for path_id, count in sorted(
for permalink, count in sorted(
entity_changes.items(),
key=lambda x: x[1],
reverse=True
)[:5]:
print(f"- {path_id}: {count} changes")
"""
print(f"- {permalink}: {count} changes")
""",
},
{
"name": "Context Building",
@@ -89,14 +89,14 @@ activity = await get_recent_activity(timeframe="1d")
# Extract changed entities for deeper analysis
entity_ids = [
change.path_id for change in activity.changes
change.permalink for change in activity.changes
if change.activity_type == "entity"
]
# Load full entity details
if entity_ids:
entities = await open_nodes(
request=OpenNodesRequest(path_ids=entity_ids)
request=OpenNodesRequest(permalinks=entity_ids)
)
# Analyze recent development focus
@@ -112,27 +112,27 @@ if entity_ids:
print(f"\\n{name}:")
for obs in observations:
print(f"- {obs.content}")
"""
}
""",
},
],
output_model=RecentActivity
output_model=RecentActivity,
)
async def get_recent_activity(
timeframe: str = "1d",
activity_types: Optional[List[ActivityType]] = None,
) -> RecentActivity:
"""Track changes across the knowledge base.
Args:
timeframe: Time window to analyze ("1h", "1d", "1w")
activity_types: Optional list of types to filter by
Returns:
RecentActivity object with changes and summary statistics
"""
logger.debug(f"Getting recent activity (timeframe={timeframe}, types={activity_types})")
# Build params
# Build params
params = {
"timeframe": timeframe,
}
@@ -141,4 +141,4 @@ async def get_recent_activity(
# Get activity
response = await client.get("/activity/recent", params=params)
return RecentActivity.model_validate(response.json())
return RecentActivity.model_validate(response.json())
src/basic_memory/mcp/tools/knowledge.py
+28 -28
View File
@@ -42,7 +42,7 @@ await create_entities({
]
}]
})
"""
""",
},
{
"name": "Create Feature",
@@ -61,10 +61,10 @@ await create_entities({
]
}]
})
"""
}
""",
},
],
output_model=EntityListResponse
output_model=EntityListResponse,
)
async def create_entities(request: CreateEntityRequest) -> EntityListResponse:
"""Create new entities in the knowledge graph."""
@@ -90,7 +90,7 @@ await create_relations({
"context": "Needs storage for search indexes"
}]
})
"""
""",
},
{
"name": "Link Implementation",
@@ -105,10 +105,10 @@ await create_relations({
"context": "Primary search implementation"
}]
})
"""
}
""",
},
],
output_model=EntityListResponse
output_model=EntityListResponse,
)
async def create_relations(request: CreateRelationsRequest) -> EntityListResponse:
"""Create relations between existing entities."""
@@ -141,29 +141,29 @@ deps = [rel for rel in component.relations
print("\\nDependencies:")
for dep in deps:
print(f"- {dep.to_id}")
"""
""",
}
],
output_model=EntityResponse
output_model=EntityResponse,
)
async def get_entity(path_id: PathId, content: bool = False) -> EntityResponse:
"""Get a specific entity by its path_id.
async def get_entity(permalink: PathId, content: bool = False) -> EntityResponse:
"""Get a specific entity by its permalink.
Args:
path_id: Path identifier for the entity
permalink: Path identifier for the entity
content: If True, includes the full markdown content of the entity
"""
try:
url = f"/knowledge/entities/{path_id}"
url = f"/knowledge/entities/{permalink}"
params = {"content": "true"} if content else {}
response = await client.get(url, params=params)
if response.status_code == 404:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
response.raise_for_status()
return EntityResponse.model_validate(response.json())
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
raise
@@ -177,7 +177,7 @@ async def get_entity(path_id: PathId, content: bool = False) -> EntityResponse:
# Add technical observations
await add_observations(
request=AddObservationsRequest(
path_id="component/search_service",
permalink="component/search_service",
context="Performance optimization",
observations=[
ObservationCreate(
@@ -195,10 +195,10 @@ await add_observations(
]
)
)
"""
""",
}
],
output_model=EntityResponse
output_model=EntityResponse,
)
async def add_observations(request: AddObservationsRequest) -> EntityResponse:
"""Add observations to an existing entity."""
@@ -217,17 +217,17 @@ async def add_observations(request: AddObservationsRequest) -> EntityResponse:
# Remove old implementation notes
await delete_observations(
request=DeleteObservationsRequest(
path_id="component/indexer",
permalink="component/indexer",
observations=[
"Using old indexing algorithm",
"Temporary workaround for issue #123"
]
)
)
"""
""",
}
],
output_model=EntityResponse
output_model=EntityResponse,
)
async def delete_observations(request: DeleteObservationsRequest) -> EntityResponse:
"""Delete specific observations from an entity."""
@@ -253,10 +253,10 @@ await delete_relations(
}]
)
)
"""
""",
}
],
output_model=EntityListResponse
output_model=EntityListResponse,
)
async def delete_relations(request: DeleteRelationsRequest) -> EntityListResponse:
"""Delete relations between entities."""
@@ -275,16 +275,16 @@ async def delete_relations(request: DeleteRelationsRequest) -> EntityListRespons
# Remove deprecated components
await delete_entities(
request=DeleteEntitiesRequest(
path_ids=[
permalinks=[
"component/old_service",
"test/obsolete_test"
]
)
)
"""
""",
}
],
output_model=Dict[str, bool]
output_model=Dict[str, bool],
)
async def delete_entities(request: DeleteEntitiesRequest) -> DeleteEntitiesResponse:
"""Delete entities from the knowledge graph."""
src/basic_memory/mcp/tools/search.py
+11 -12
View File
@@ -28,7 +28,7 @@ by_type = defaultdict(list)
for result in results.results:
meta = result.metadata
path = result.path_id
path = result.permalink
# Group by status if available
if "status" in meta:
@@ -78,7 +78,7 @@ sorted_results = sorted(
print("Recent Changes:")
for result in sorted_results:
print(f"\\n{result.path_id}")
print(f"\\n{result.permalink}")
print(f"Type: {result.type}")
print(f"Score: {result.score:.2f}")
if "updated_at" in result.metadata:
@@ -112,7 +112,7 @@ docs.sort(key=lambda x: x.score)
print("Technical Documentation:")
for doc in docs:
print(f"\\n{doc.path_id}")
print(f"\\n{doc.permalink}")
if "title" in doc.metadata:
print(f"Title: {doc.metadata['title']}")
print(f"Score: {doc.score:.2f}")
@@ -125,7 +125,7 @@ for doc in docs:
"description": "Find content related to a specific entity",
"code": """
# First get the entity to extract key terms
entity = await get_entity(path_id="component/memory_service")
entity = await get_entity(permalink="component/memory_service")
if entity:
# Build search terms from entity info
@@ -143,12 +143,12 @@ if entity:
)
# Filter out the original entity and sort by relevance
related = [r for r in results.results if r.path_id != entity["path_id"]]
related = [r for r in results.results if r.permalink != entity["permalink"]]
related.sort(key=lambda x: x.score)
print(f"Content Related to {entity['name']}:")
for result in related[:5]: # Top 5 most relevant
print(f"\\n{result.path_id}")
print(f"\\n{result.permalink}")
print(f"Type: {result.type}")
print(f"Score: {result.score:.2f}")
""",
@@ -156,7 +156,6 @@ if entity:
],
)
async def search(query: SearchQuery) -> SearchResponse:
"""Search across all content in basic-memory.
Args:
@@ -175,7 +174,7 @@ async def search(query: SearchQuery) -> SearchResponse:
@mcp.tool(
category="search",
description="Load multiple entities by their path_ids in a single request",
description="Load multiple entities by their permalinks in a single request",
examples=[
{
"name": "Load and Analyze Entity Context",
@@ -192,9 +191,9 @@ results = await search(
if results.results:
# Load full context for found entities
path_ids = [r.path_id for r in results.results]
permalinks = [r.permalink for r in results.results]
context = await open_nodes(
request=OpenNodesRequest(path_ids=path_ids)
request=OpenNodesRequest(permalinks=permalinks)
)
# Analyze relationships
@@ -219,10 +218,10 @@ if results.results:
],
)
async def open_nodes(request: OpenNodesRequest) -> EntityListResponse:
"""Load multiple entities by their path_ids.
"""Load multiple entities by their permalinks.
Args:
request: OpenNodesRequest containing list of path_ids to load
request: OpenNodesRequest containing list of permalinks to load
Returns:
EntityListResponse containing complete details for each requested entity
src/basic_memory/models/knowledge.py
+7 -10
View File
@@ -13,7 +13,6 @@ from sqlalchemy import (
DateTime,
Index,
JSON,
CheckConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -27,14 +26,14 @@ class Entity(Base):
Entities represent semantic nodes maintained by the AI layer. Each entity:
- Has a unique numeric ID (database-generated)
- Maps to a file on disk
- Maps to a file on disk
- Maintains a checksum for change detection
- Tracks both source file and semantic properties
"""
__tablename__ = "entity"
__table_args__ = (
UniqueConstraint("path_id", name="uix_entity_path_id"), # Make path_id unique
UniqueConstraint("permalink", name="uix_entity_permalink"), # Make permalink unique
Index("ix_entity_type", "entity_type"),
Index("ix_entity_created_at", "created_at"), # For timeline queries
Index("ix_entity_updated_at", "updated_at"), # For timeline queries
@@ -48,12 +47,12 @@ class Entity(Base):
content_type: Mapped[str] = mapped_column(String)
# Normalized path for URIs
path_id: Mapped[str] = mapped_column(String, unique=True, index=True)
permalink: Mapped[str] = mapped_column(String, unique=True, index=True)
# Actual filesystem relative path
file_path: Mapped[str] = mapped_column(String, unique=True, index=True)
# checksum of file
checksum: Mapped[Optional[str]] = mapped_column(String, nullable=True)
# Content summary
summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
@@ -96,6 +95,7 @@ class ObservationCategory(str, Enum):
ISSUE = "issue"
TODO = "todo"
class Observation(Base):
"""
An observation about an entity.
@@ -122,12 +122,9 @@ class Observation(Base):
)
context: Mapped[str] = mapped_column(Text, nullable=True)
tags: Mapped[Optional[list[str]]] = mapped_column(
JSON,
nullable=True,
default=list,
server_default='[]'
JSON, nullable=True, default=list, server_default="[]"
)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP")
src/basic_memory/models/search.py
+2 -2
View File
@@ -6,10 +6,10 @@ from sqlalchemy import DDL
CREATE_SEARCH_INDEX = DDL("""
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
content, -- Searchable text content
path_id UNINDEXED, -- Link to entity/document (must be unique)
permalink UNINDEXED, -- Link to entity/document (must be unique)
file_path UNINDEXED, -- Filesystem path
type UNINDEXED, -- 'entity' or 'document'
metadata UNINDEXED, -- JSON with timestamps, types, etc.
tokenize='porter unicode61' -- Enable stemming + unicode
);
""")
""")
src/basic_memory/repository/entity_repository.py
+24 -18
View File
@@ -2,7 +2,7 @@
from typing import List, Optional, Sequence
from sqlalchemy import select, or_, asc, desc
from sqlalchemy import select, or_, asc
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
@@ -18,16 +18,16 @@ class EntityRepository(Repository[Entity]):
"""Initialize with session maker."""
super().__init__(session_maker, Entity)
async def get_by_path_id(self, path_id: str) -> Optional[Entity]:
"""Get entity by path_id."""
query = self.select().where(Entity.path_id == path_id).options(*self.get_load_options())
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
"""Get entity by permalink."""
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
return await self.find_one(query)
async def list_entities(
self,
entity_type: Optional[str] = None,
sort_by: Optional[str] = "updated_at",
include_related: bool = False,
self,
entity_type: Optional[str] = None,
sort_by: Optional[str] = "updated_at",
include_related: bool = False,
) -> Sequence[Entity]:
"""List all entities, optionally filtered by type and sorted."""
query = self.select()
@@ -44,8 +44,12 @@ class EntityRepository(Repository[Entity]):
query = query.where(
or_(
Entity.entity_type == entity_type,
Entity.outgoing_relations.any(Relation.to_entity.has(entity_type=entity_type)),
Entity.incoming_relations.any(Relation.from_entity.has(entity_type=entity_type))
Entity.outgoing_relations.any(
Relation.to_entity.has(entity_type=entity_type)
),
Entity.incoming_relations.any(
Relation.from_entity.has(entity_type=entity_type)
),
)
)
else:
@@ -111,28 +115,30 @@ class EntityRepository(Repository[Entity]):
selectinload(Entity.incoming_relations).selectinload(Relation.to_entity),
]
async def find_by_path_ids(self, path_ids: List[str]) -> Sequence[Entity]:
"""Find multiple entities by their path_id."""
async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
"""Find multiple entities by their permalink."""
# Handle empty input explicitly
if not path_ids:
if not permalinks:
return []
# Use existing select pattern
query = self.select().options(*self.get_load_options()).where(Entity.path_id.in_(path_ids))
query = (
self.select().options(*self.get_load_options()).where(Entity.permalink.in_(permalinks))
)
result = await self.execute_query(query)
return list(result.scalars().all())
async def delete_by_path_ids(self, path_ids: List[str]) -> int:
"""Delete multiple entities by path_id."""
async def delete_by_permalinks(self, permalinks: List[str]) -> int:
"""Delete multiple entities by permalink."""
# Handle empty input explicitly
if not path_ids:
if not permalinks:
return 0
# Find matching entities
entities = await self.find_by_path_ids(path_ids)
entities = await self.find_by_permalinks(permalinks)
if not entities:
return 0
src/basic_memory/repository/relation_repository.py
+8 -7
View File
@@ -1,4 +1,5 @@
"""Repository for managing Relation objects."""
from sqlalchemy import and_, delete
from typing import Sequence, List, Optional
@@ -18,7 +19,9 @@ class RelationRepository(Repository[Relation]):
def __init__(self, session_maker: async_sessionmaker):
super().__init__(session_maker, Relation)
async def find_relation(self, from_path_id: str, to_path_id: str, relation_type: str) -> Optional[Relation]:
async def find_relation(
self, from_permalink: str, to_permalink: str, relation_type: str
) -> Optional[Relation]:
"""Find a relation by its from and to path IDs."""
from_entity = aliased(Entity)
to_entity = aliased(Entity)
@@ -29,9 +32,9 @@ class RelationRepository(Repository[Relation]):
.join(to_entity, Relation.to_id == to_entity.id)
.where(
and_(
from_entity.path_id == from_path_id,
to_entity.path_id == to_path_id,
Relation.relation_type == relation_type
from_entity.permalink == from_permalink,
to_entity.permalink == to_permalink,
Relation.relation_type == relation_type,
)
)
)
@@ -62,9 +65,7 @@ class RelationRepository(Repository[Relation]):
as these are the ones owned by this entity's markdown file.
"""
async with db.scoped_session(self.session_maker) as session:
await session.execute(
delete(Relation).where(Relation.from_id == entity_id)
)
await session.execute(delete(Relation).where(Relation.from_id == entity_id))
def get_load_options(self) -> List[LoaderOption]:
return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]
src/basic_memory/repository/search_repository.py
+22 -30
View File
@@ -2,19 +2,17 @@
import json
from typing import List, Optional
from datetime import datetime
from loguru import logger
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from basic_memory import db
from basic_memory.repository.repository import Repository
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
from basic_memory.models.search import CREATE_SEARCH_INDEX
class SearchRepository():
class SearchRepository:
"""Repository for search index operations."""
def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
@@ -27,9 +25,7 @@ class SearchRepository():
await session.commit()
async def search(
self,
query: SearchQuery,
context: Optional[List[str]] = None
self, query: SearchQuery, context: Optional[List[str]] = None
) -> List[SearchResult]:
"""Search across all indexed content."""
conditions = []
@@ -48,23 +44,19 @@ class SearchRepository():
# Handle entity type filter
if query.entity_types:
entity_type_list = ", ".join(f"'{t}'" for t in query.entity_types)
conditions.append(
f"json_extract(metadata, '$.entity_type') IN ({entity_type_list})"
)
conditions.append(f"json_extract(metadata, '$.entity_type') IN ({entity_type_list})")
# Handle date filter
if query.after_date:
params["after_date"] = query.after_date
conditions.append(
"json_extract(metadata, '$.created_at') > :after_date"
)
conditions.append("json_extract(metadata, '$.created_at') > :after_date")
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
sql = f"""
SELECT
path_id,
permalink,
file_path,
type,
metadata,
@@ -80,11 +72,11 @@ class SearchRepository():
return [
SearchResult(
path_id=row.path_id,
permalink=row.permalink,
file_path=row.file_path,
type=SearchItemType(row.type), # Convert string to enum
score=row.score,
metadata=json.loads(row.metadata)
metadata=json.loads(row.metadata),
)
for row in rows
]
@@ -92,44 +84,44 @@ class SearchRepository():
async def index_item(
self,
content: str,
path_id: str,
permalink: str,
file_path: str,
type: SearchItemType, # Now accepts enum
metadata: dict
metadata: dict,
):
"""Index or update a single item."""
async with db.scoped_session(self.session_maker) as session:
# Delete existing record if any
await session.execute(
text("DELETE FROM search_index WHERE path_id = :path_id"),
{"path_id": path_id}
text("DELETE FROM search_index WHERE permalink = :permalink"),
{"permalink": permalink},
)
# Insert new record
await session.execute(
text("""
INSERT INTO search_index (
content, path_id, file_path, type, metadata
content, permalink, file_path, type, metadata
) VALUES (
:content, :path_id, :file_path, :type, :metadata
:content, :permalink, :file_path, :type, :metadata
)
"""),
{
"content": content,
"path_id": path_id,
"permalink": permalink,
"file_path": file_path,
"type": type.value, # Store the string value
"metadata": json.dumps(metadata)
}
"metadata": json.dumps(metadata),
},
)
logger.debug(f"indexed {path_id}")
logger.debug(f"indexed {permalink}")
await session.commit()
async def delete_by_path_id(self, path_id: str):
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
async with db.scoped_session(self.session_maker) as session:
await session.execute(
text("DELETE FROM search_index WHERE path_id = :path_id"),
{"path_id": path_id}
text("DELETE FROM search_index WHERE permalink = :permalink"),
{"permalink": permalink},
)
await session.commit()
await session.commit()
src/basic_memory/schemas/activity.py
+17 -13
View File
@@ -7,34 +7,34 @@ from pydantic import BaseModel, Field
class TimeFrame:
"""Represents a time period for querying activity."""
def __init__(self, timeframe_str: str):
"""Parse timeframe string (e.g., '1d', '2w', '1m')"""
if not timeframe_str or len(timeframe_str) < 2:
raise ValueError("Invalid timeframe format")
try:
self.value = int(timeframe_str[:-1])
self.unit = timeframe_str[-1]
except ValueError:
raise ValueError("Invalid timeframe format")
if self.unit not in ['h', 'd', 'w', 'm']:
if self.unit not in ["h", "d", "w", "m"]:
raise ValueError("Invalid timeframe unit")
if self.value < 1:
raise ValueError("Timeframe value must be positive")
@property
def to_timedelta(self) -> timedelta:
"""Convert to Python timedelta."""
if self.unit == 'h':
if self.unit == "h":
return timedelta(hours=self.value)
elif self.unit == 'd':
elif self.unit == "d":
return timedelta(days=self.value)
elif self.unit == 'w':
elif self.unit == "w":
return timedelta(weeks=self.value)
elif self.unit == 'm':
elif self.unit == "m":
# Approximate month as 30 days
return timedelta(days=self.value * 30)
else:
@@ -43,12 +43,14 @@ class TimeFrame:
class ActivityType(str, Enum):
"""Types of activities that can be tracked."""
ENTITY = "entity"
RELATION = "relation"
class ChangeType(str, Enum):
"""Types of changes that can occur."""
CREATED = "created"
UPDATED = "updated"
DELETED = "deleted"
@@ -56,26 +58,28 @@ class ChangeType(str, Enum):
class ActivityChange(BaseModel):
"""Represents a single change in the system."""
activity_type: ActivityType
change_type: ChangeType
timestamp: datetime
path_id: str
permalink: str
summary: str
content: Optional[str] = None
class ActivitySummary(BaseModel):
"""Summary statistics about recent activity."""
entity_changes: int = Field(default=0, description="Number of entity changes")
relation_changes: int = Field(default=0, description="Number of relation changes")
most_active_paths: List[str] = Field(
default_factory=list,
description="List of most frequently changed paths"
default_factory=list, description="List of most frequently changed paths"
)
class RecentActivity(BaseModel):
"""Complete activity report."""
timeframe: str
changes: List[ActivityChange] = Field(default_factory=list)
summary: ActivitySummary
src/basic_memory/schemas/base.py
+30 -30
View File
@@ -10,6 +10,7 @@ Key Concepts:
3. Observations are atomic facts/notes about an entity
4. Everything is stored in both SQLite and markdown files
"""
import mimetypes
import re
from enum import Enum
@@ -29,7 +30,7 @@ def to_snake_case(name: str) -> str:
Memory_Service -> memory_service
"""
name = name.strip()
# Replace spaces and hyphens and . with underscores
s1 = re.sub(r"[\s\-\\.]", "_", name)
@@ -48,7 +49,6 @@ def validate_path_format(path: str) -> str:
return path
class ObservationCategory(str, Enum):
"""Categories for structuring observations.
@@ -62,6 +62,7 @@ class ObservationCategory(str, Enum):
Categories are case-insensitive for easier use.
"""
TECH = "tech"
DESIGN = "design"
FEATURE = "feature"
@@ -76,16 +77,16 @@ class ObservationCategory(str, Enum):
return cls(value.lower())
except ValueError:
return None
PathId = Annotated[str, BeforeValidator(to_snake_case), BeforeValidator(validate_path_format)]
"""Unique identifier in format '{path}/{normalized_name}'."""
Observation = Annotated[
str,
str,
BeforeValidator(str.strip), # Clean whitespace
MinLen(1), # Ensure non-empty after stripping
MaxLen(1000) # Keep reasonable length
MaxLen(1000), # Keep reasonable length
]
"""A single piece of information about an entity. Must be non-empty and under 1000 characters.
"""
@@ -94,23 +95,22 @@ EntityType = Annotated[str, BeforeValidator(to_snake_case), MinLen(1), MaxLen(20
"""Classification of entity (e.g., 'person', 'project', 'concept'). """
ALLOWED_CONTENT_TYPES = {
'text/markdown',
'text/plain',
'application/pdf',
'image/jpeg',
'image/png',
'image/svg+xml',
"text/markdown",
"text/plain",
"application/pdf",
"image/jpeg",
"image/png",
"image/svg+xml",
}
ContentType = Annotated[
str,
str,
BeforeValidator(str.lower),
Field(pattern=r'^[\w\-\+\.]+/[\w\-\+\.]+$'),
Field(json_schema_extra={"examples": list(ALLOWED_CONTENT_TYPES)})
Field(pattern=r"^[\w\-\+\.]+/[\w\-\+\.]+$"),
Field(json_schema_extra={"examples": list(ALLOWED_CONTENT_TYPES)}),
]
RelationType = Annotated[str, BeforeValidator(to_snake_case), MinLen(1), MaxLen(200)]
"""Type of relationship between entities. Always use active voice present tense."""
@@ -119,7 +119,7 @@ class Relation(BaseModel):
"""Represents a directed edge between entities in the knowledge graph.
Relations are directed connections stored in active voice (e.g., "created", "depends_on").
The from_path_id represents the source or actor entity, while to_path_id represents the target
The from_permalink represents the source or actor entity, while to_permalink represents the target
or recipient entity.
"""
@@ -133,7 +133,7 @@ class Entity(BaseModel):
"""Represents a node in our knowledge graph - could be a person, project, concept, etc.
Each entity has:
- A title
- A title
- An entity type (for classification)
- A list of observations (facts/notes about the entity)
- Optional relations to other entities
@@ -147,32 +147,32 @@ class Entity(BaseModel):
summary: Optional[str] = None
content_type: ContentType = Field(
description="MIME type of the content (e.g. text/markdown, image/jpeg)",
examples=["text/markdown", "image/jpeg"]
)
examples=["text/markdown", "image/jpeg"],
)
observations: List[Observation] = []
@property
def path_id(self) -> PathId:
def permalink(self) -> PathId:
"""Get the path ID in format {snake_case_title}."""
normalized_name = to_snake_case(self.title)
return normalized_name
@property
def file_path(self):
"""Get the file path for this entity based on its path_id."""
return f"{self.path_id}.md"
"""Get the file path for this entity based on its permalink."""
return f"{self.permalink}.md"
@model_validator(mode='before')
@model_validator(mode="before")
@classmethod
def infer_content_type(cls, data: Dict) -> Dict:
"""Infer content_type from file_path if not provided."""
if 'content_type' not in data:
# Get path from either file_path or construct from path_id
file_path = data.get('file_path') or f"{data.get('name')}.md"
if "content_type" not in data:
# Get path from either file_path or construct from permalink
file_path = data.get("file_path") or f"{data.get('name')}.md"
if not file_path:
raise ValidationError("Either file_path or name must be provided")
mime_type, _ = mimetypes.guess_type(file_path)
data['content_type'] = mime_type or 'text/plain'
data["content_type"] = mime_type or "text/plain"
return data
return data
src/basic_memory/schemas/delete.py
+2 -2
View File
@@ -34,7 +34,7 @@ class DeleteEntitiesRequest(BaseModel):
4. Deletes the corresponding markdown file
"""
path_ids: Annotated[List[PathId], MinLen(1)]
permalinks: Annotated[List[PathId], MinLen(1)]
class DeleteRelationsRequest(BaseModel):
@@ -56,5 +56,5 @@ class DeleteObservationsRequest(BaseModel):
match exactly for deletion.
"""
path_id: PathId
permalink: PathId
observations: Annotated[List[Observation], MinLen(1)]
src/basic_memory/schemas/request.py
+15 -8
View File
@@ -3,16 +3,23 @@
from typing import List, Optional, Annotated, Dict, Any
from annotated_types import MaxLen, MinLen
from pydantic import BaseModel, StringConstraints
from pydantic import BaseModel
from basic_memory.schemas.base import Observation, Entity, Relation, PathId, ObservationCategory, EntityType
from basic_memory.schemas.base import (
Observation,
Entity,
Relation,
PathId,
ObservationCategory,
EntityType,
)
class ObservationCreate(BaseModel):
"""A single observation with category, content, and optional context."""
category: ObservationCategory = ObservationCategory.NOTE
content: Observation
content: Observation
class AddObservationsRequest(BaseModel):
@@ -23,7 +30,7 @@ class AddObservationsRequest(BaseModel):
to our understanding of the entity.
"""
path_id: PathId
permalink: PathId
context: Optional[str] = None
observations: List[ObservationCreate]
@@ -34,7 +41,7 @@ class CreateEntityRequest(BaseModel):
Entities represent nodes in the knowledge graph. They can be created
with initial observations and optional descriptions. Entity IDs are
automatically generated from the type and name.
Observations will be assigned the default category of 'note'.
"""
@@ -81,21 +88,21 @@ class OpenNodesRequest(BaseModel):
discovered through search.
"""
path_ids: Annotated[List[PathId], MinLen(1)]
permalinks: Annotated[List[PathId], MinLen(1)]
class CreateRelationsRequest(BaseModel):
relations: List[Relation]
## update
class UpdateEntityRequest(BaseModel):
"""Request to update an existing entity."""
title: Optional[str] = None
entity_type: Optional[EntityType] = None
summary: Optional[str] = None
content: Optional[str] = None
entity_metadata: Optional[Dict[str, Any]] = None
src/basic_memory/schemas/response.py
+17 -18
View File
@@ -11,12 +11,11 @@ Key Features:
4. Bulk operations return all affected items
"""
import datetime
from typing import List, Optional, Dict, Any
from typing import List, Optional, Dict
from pydantic import BaseModel, ConfigDict, Field, AliasPath, AliasChoices
from basic_memory.schemas.base import Observation, Relation, PathId, Entity, EntityType, ContentType
from basic_memory.schemas.base import Relation, PathId, EntityType, ContentType
from basic_memory.schemas.request import ObservationCreate
@@ -44,6 +43,7 @@ class ObservationResponse(ObservationCreate, SQLAlchemyModel):
"context": "Initial database design meeting"
}
"""
context: Optional[str] = None
@@ -61,20 +61,21 @@ class RelationResponse(Relation, SQLAlchemyModel):
"context": "Comprehensive test suite"
}
"""
from_id: PathId = Field(
# use the path_id from the associated Entity
# use the permalink from the associated Entity
# or the from_id value
validation_alias=AliasChoices(
AliasPath('from_entity', 'path_id'),
'from_id',
AliasPath("from_entity", "permalink"),
"from_id",
)
)
to_id: PathId = Field(
# use the path_id from the associated Entity
# use the permalink from the associated Entity
# or the to_id value
validation_alias=AliasChoices(
AliasPath('to_entity', 'path_id'),
'to_id',
AliasPath("to_entity", "permalink"),
"to_id",
)
)
@@ -90,7 +91,7 @@ class EntityResponse(SQLAlchemyModel):
Example Response:
{
"path_id": "component/memory_service",
"permalink": "component/memory_service",
"title": "MemoryService",
"entity_type": "component",
"description": "Core persistence service",
@@ -117,8 +118,8 @@ class EntityResponse(SQLAlchemyModel):
}
"""
# Note this Class does not inherit form Entity because of the Entity.path_id semantics
path_id: PathId
# Note this Class does not inherit form Entity because of the Entity.permalink semantics
permalink: PathId
title: str
entity_type: EntityType
entity_metadata: Optional[Dict] = None
@@ -133,14 +134,14 @@ class EntityListResponse(SQLAlchemyModel):
"""Response for create_entities operation.
Returns complete information about entities returned from the service,
including their path_ids, observations,
including their permalinks, observations,
and any established relations.
Example Response:
{
"entities": [
{
"path_id": "component/search_service",
"permalink": "component/search_service",
"title": "SearchService",
"entity_type": "component",
"description": "Knowledge graph search",
@@ -152,7 +153,7 @@ class EntityListResponse(SQLAlchemyModel):
"relations": []
},
{
"path_id": "document/api_docs",
"permalink": "document/api_docs",
"title": "API_Documentation",
"entity_type": "document",
"description": "API Reference",
@@ -180,7 +181,7 @@ class SearchNodesResponse(SQLAlchemyModel):
{
"matches": [
{
"path_id": "component/memory_service",
"permalink": "component/memory_service",
"title": "MemoryService",
"entity_type": "component",
"description": "Core service",
@@ -199,7 +200,6 @@ class SearchNodesResponse(SQLAlchemyModel):
query: str
class DeleteEntitiesResponse(SQLAlchemyModel):
"""Response indicating successful entity deletion.
@@ -213,4 +213,3 @@ class DeleteEntitiesResponse(SQLAlchemyModel):
"""
deleted: bool
src/basic_memory/schemas/search.py
+7 -3
View File
@@ -8,18 +8,20 @@ from pydantic import BaseModel, field_validator
class SearchItemType(str, Enum):
"""Types of searchable items."""
DOCUMENT = "document"
ENTITY = "entity"
class SearchQuery(BaseModel):
"""Search query parameters."""
text: str
types: Optional[List[SearchItemType]] = None
entity_types: Optional[List[str]] = None
after_date: Optional[Union[datetime, str]] = None
@field_validator('after_date')
@field_validator("after_date")
@classmethod
def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]:
"""Convert datetime to ISO format if needed."""
@@ -32,7 +34,8 @@ class SearchQuery(BaseModel):
class SearchResult(BaseModel):
"""Search result item."""
path_id: str
permalink: str
file_path: str
type: SearchItemType
score: float
@@ -41,4 +44,5 @@ class SearchResult(BaseModel):
class SearchResponse(BaseModel):
"""Wrapper for search results list."""
results: List[SearchResult]
results: List[SearchResult]
src/basic_memory/services/activity_service.py
+43 -34
View File
@@ -1,7 +1,7 @@
"""Service for tracking and querying activity across the knowledge base."""
from datetime import datetime, timezone
from typing import List, Optional, Sequence
from typing import List, Optional
from . import EntityService, RelationService
from ..schemas.activity import (
@@ -32,11 +32,11 @@ class ActivityService:
activity_types: Optional[List[str]] = None,
) -> RecentActivity:
"""Get all recent activity in the knowledge base.
Args:
timeframe: Time window to look back (1h, 1d, 1w, 1m)
activity_types: Optional list of types to include
Returns:
RecentActivity object containing changes and summary
"""
@@ -47,9 +47,7 @@ class ActivityService:
# Get changes based on requested types
changes = []
types_to_fetch = (
[ActivityType(t) for t in activity_types]
if activity_types
else list(ActivityType)
[ActivityType(t) for t in activity_types] if activity_types else list(ActivityType)
)
for activity_type in types_to_fetch:
@@ -68,81 +66,92 @@ class ActivityService:
summary = ActivitySummary(
entity_changes=len([c for c in changes if c.activity_type == ActivityType.ENTITY]),
relation_changes=len([c for c in changes if c.activity_type == ActivityType.RELATION]),
most_active_paths=self._get_most_active_paths(changes)
most_active_paths=self._get_most_active_paths(changes),
)
return RecentActivity(
timeframe=timeframe,
changes=changes,
summary=summary
)
return RecentActivity(timeframe=timeframe, changes=changes, summary=summary)
async def _get_entity_changes(self, since: datetime) -> List[ActivityChange]:
"""Get recent entity changes."""
# Query entities updated since the cutoff
entities = await self.entity_service.get_modified_since(since)
changes = []
for entity in entities:
# Ensure timestamps are timezone-aware
created_at = entity.created_at.replace(tzinfo=timezone.utc) if entity.created_at.tzinfo is None else entity.created_at
updated_at = entity.updated_at.replace(tzinfo=timezone.utc) if entity.updated_at.tzinfo is None else entity.updated_at
created_at = (
entity.created_at.replace(tzinfo=timezone.utc)
if entity.created_at.tzinfo is None
else entity.created_at
)
updated_at = (
entity.updated_at.replace(tzinfo=timezone.utc)
if entity.updated_at.tzinfo is None
else entity.updated_at
)
change_type = ChangeType.CREATED if created_at >= since else ChangeType.UPDATED
changes.append(
ActivityChange(
activity_type=ActivityType.ENTITY,
change_type=change_type,
timestamp=updated_at,
path_id=entity.path_id,
permalink=entity.permalink,
summary=f"{change_type.value.title()} entity: {entity.title}",
content=entity.summary
content=entity.summary,
)
)
return changes
return changes
async def _get_relation_changes(self, since: datetime) -> List[ActivityChange]:
"""Get recent relation changes."""
# Query relations updated since the cutoff
relations = await self.relation_service.get_modified_since(since)
changes = []
for relation in relations:
# Ensure timestamps are timezone-aware
created_at = relation.created_at.replace(tzinfo=timezone.utc) if relation.created_at.tzinfo is None else relation.created_at
updated_at = relation.updated_at.replace(tzinfo=timezone.utc) if relation.updated_at.tzinfo is None else relation.updated_at
created_at = (
relation.created_at.replace(tzinfo=timezone.utc)
if relation.created_at.tzinfo is None
else relation.created_at
)
updated_at = (
relation.updated_at.replace(tzinfo=timezone.utc)
if relation.updated_at.tzinfo is None
else relation.updated_at
)
change_type = ChangeType.CREATED if created_at >= since else ChangeType.UPDATED
changes.append(
ActivityChange(
activity_type=ActivityType.RELATION,
change_type=change_type,
timestamp=updated_at,
path_id=f"{relation.from_id}->{relation.to_id}",
permalink=f"{relation.from_id}->{relation.to_id}",
summary=(
f"{change_type.value.title()} relation: "
f"{relation.from_id} {relation.relation_type} {relation.to_id}"
),
content=relation.context
content=relation.context,
)
)
return changes
def _get_most_active_paths(self, changes: List[ActivityChange], limit: int = 5) -> List[str]:
"""Get the most frequently changed paths."""
path_counts = {}
for change in changes:
path_counts[change.path_id] = path_counts.get(change.path_id, 0) + 1
path_counts[change.permalink] = path_counts.get(change.permalink, 0) + 1
# Sort by count descending and take top paths
sorted_paths = sorted(
path_counts.items(),
key=lambda x: (-x[1], x[0]) # Sort by count desc, then path asc
key=lambda x: (-x[1], x[0]), # Sort by count desc, then path asc
)
return [path for path, _ in sorted_paths[:limit]]
return [path for path, _ in sorted_paths[:limit]]
src/basic_memory/services/entity_service.py
+22 -22
View File
@@ -17,7 +17,7 @@ def entity_model(entity: EntitySchema):
title=entity.title,
entity_type=entity.entity_type,
entity_metadata=entity.entity_metadata,
path_id=entity.path_id,
permalink=entity.permalink,
file_path=entity.file_path,
summary=entity.summary,
content_type=entity.content_type,
@@ -57,7 +57,7 @@ class EntityService(BaseService[EntityModel]):
except Exception as e:
# Clean up on any failure
if db_entity:
await self.delete_entity(db_entity.path_id)
await self.delete_entity(db_entity.permalink)
await self.file_service.delete_entity_file(db_entity)
logger.error(f"Failed to create entity: {e}")
raise
@@ -69,7 +69,7 @@ class EntityService(BaseService[EntityModel]):
async def update_entity(
self,
path_id: str,
permalink: str,
content: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**update_fields: Any,
@@ -77,7 +77,7 @@ class EntityService(BaseService[EntityModel]):
"""Update an entity's content and metadata.
Args:
path_id: Entity's path ID
permalink: Entity's path ID
content: Optional new content
metadata: Optional metadata updates
**update_fields: Additional entity fields to update
@@ -88,12 +88,12 @@ class EntityService(BaseService[EntityModel]):
Raises:
EntityNotFoundError: If entity doesn't exist
"""
logger.debug(f"Updating entity with path_id: {path_id}")
logger.debug(f"Updating entity with permalink: {permalink}")
# Get existing entity
entity = await self.get_by_path_id(path_id)
entity = await self.get_by_permalink(permalink)
if not entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
try:
# Build update data
@@ -128,13 +128,13 @@ class EntityService(BaseService[EntityModel]):
logger.error(f"Failed to update entity: {e}")
raise
async def delete_entity(self, path_id: str) -> bool:
async def delete_entity(self, permalink: str) -> bool:
"""Delete entity and its file."""
logger.debug(f"Deleting entity: {path_id}")
logger.debug(f"Deleting entity: {permalink}")
try:
# Get entity first for file deletion
entity = await self.get_by_path_id(path_id)
entity = await self.get_by_permalink(permalink)
# Delete file first (it's source of truth)
await self.file_service.delete_entity_file(entity)
@@ -143,30 +143,30 @@ class EntityService(BaseService[EntityModel]):
return await self.repository.delete(entity.id)
except EntityNotFoundError:
logger.info(f"Entity not found: {path_id}")
logger.info(f"Entity not found: {permalink}")
return True # Already deleted
except Exception as e:
logger.error(f"Failed to delete entity: {e}")
raise
async def delete_entities(self, path_ids: List[str]) -> bool:
async def delete_entities(self, permalinks: List[str]) -> bool:
"""Delete multiple entities and their files."""
logger.debug(f"Deleting entities: {path_ids}")
logger.debug(f"Deleting entities: {permalinks}")
success = True
for path_id in path_ids:
await self.delete_entity(path_id)
for permalink in permalinks:
await self.delete_entity(permalink)
success = True
return success
async def get_by_path_id(self, path_id: str) -> EntityModel:
async def get_by_permalink(self, permalink: str) -> EntityModel:
"""Get entity by type and name combination."""
logger.debug(f"Getting entity by path_id: {path_id}")
db_entity = await self.repository.get_by_path_id(path_id)
logger.debug(f"Getting entity by permalink: {permalink}")
db_entity = await self.repository.get_by_permalink(permalink)
if not db_entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
return db_entity
async def get_all(self) -> Sequence[EntityModel]:
@@ -188,10 +188,10 @@ class EntityService(BaseService[EntityModel]):
logger.debug(f"Listing entities: type={entity_type} sort={sort_by}")
return await self.repository.list_entities(entity_type=entity_type, sort_by=sort_by)
async def open_nodes(self, path_ids: List[str]) -> Sequence[EntityModel]:
async def open_nodes(self, permalinks: List[str]) -> Sequence[EntityModel]:
"""Get specific nodes and their relationships."""
logger.debug(f"Opening nodes path_ids: {path_ids}")
return await self.repository.find_by_path_ids(path_ids)
logger.debug(f"Opening nodes permalinks: {permalinks}")
return await self.repository.find_by_permalinks(permalinks)
async def delete_entity_by_file_path(self, file_path):
await self.repository.delete_by_file_path(file_path)
src/basic_memory/services/file_service.py
+4 -9
View File
@@ -1,6 +1,5 @@
"""Service for file operations with checksum tracking."""
from datetime import datetime, UTC
from pathlib import Path
from typing import Optional, Dict, Any, Tuple
@@ -31,12 +30,11 @@ class FileService:
self.base_path = base_path
self.knowledge_writer = knowledge_writer
def get_entity_path(self, entity: EntityModel) -> Path:
"""Generate filesystem path for entity."""
if entity.file_path:
return self.base_path / entity.file_path
return self.base_path / f"{entity.path_id}.md"
return self.base_path / f"{entity.permalink}.md"
async def write_entity_file(
self,
@@ -62,12 +60,11 @@ class FileService:
logger.error(f"Failed to write entity file: {e}")
raise FileOperationError(f"Failed to write entity file: {e}")
async def read_entity_content(self, entity: EntityModel) -> str:
"""Get entity's content if it's a note.
Args:
path_id: Entity's path ID
permalink: Entity's path ID
Returns:
content without frontmatter
@@ -75,7 +72,7 @@ class FileService:
Raises:
FileOperationError: If entity file doesn't exist
"""
logger.debug(f"Reading entity with path_id: {entity.path_id}")
logger.debug(f"Reading entity with permalink: {entity.permalink}")
# For notes, read the actual file content
file_path = self.get_entity_path(entity)
@@ -86,7 +83,6 @@ class FileService:
content = content.strip()
return content
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem."""
try:
@@ -96,7 +92,6 @@ class FileService:
logger.error(f"Failed to delete entity file: {e}")
raise FileOperationError(f"Failed to delete entity file: {e}")
async def exists(self, path: Path) -> bool:
"""
Check if file exists.
@@ -265,7 +260,7 @@ class FileService:
) -> str:
"""
Add YAML frontmatter to content.
Args:
content: Content to add frontmatter to
frontmatter: Frontmatter to add
src/basic_memory/services/observation_service.py
+13 -13
View File
@@ -32,7 +32,7 @@ class ObservationService(BaseService[ObservationRepository]):
self.file_operations = file_service
async def add_observations(
self, path_id: str, observations: List[ObservationCreate], context: str | None = None
self, permalink: str, observations: List[ObservationCreate], context: str | None = None
) -> EntityModel:
"""Add observations to entity and update its file.
@@ -41,17 +41,17 @@ class ObservationService(BaseService[ObservationRepository]):
- [category] Content text #tag1 #tag2 (optional context)
Args:
path_id: Entity path ID
permalink: Entity path ID
observations: List of observations with categories
context: Optional shared context for all observations
"""
logger.debug(f"Adding observations to entity: {path_id}")
logger.debug(f"Adding observations to entity: {permalink}")
try:
# Get entity to update
entity = await self.entity_repository.get_by_path_id(path_id)
entity = await self.entity_repository.get_by_permalink(permalink)
if not entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
# Add observations to DB
await self.repository.create_all(
@@ -68,33 +68,33 @@ class ObservationService(BaseService[ObservationRepository]):
)
# Get updated entity
entity = await self.entity_repository.get_by_path_id(path_id)
entity = await self.entity_repository.get_by_permalink(permalink)
# Write updated file and checksum
_, checksum = await self.file_operations.write_entity_file(entity)
await self.entity_repository.update(entity.id, {"checksum": checksum})
# Return final entity with all updates and relations
return await self.entity_repository.get_by_path_id(path_id)
return await self.entity_repository.get_by_permalink(permalink)
except Exception as e:
logger.error(f"Failed to add observations: {e}")
raise
async def delete_observations(self, path_id: str, observations: List[str]) -> EntityModel:
async def delete_observations(self, permalink: str, observations: List[str]) -> EntityModel:
"""Delete observations from entity and update its file.
Args:
path_id: Entity path ID
permalink: Entity path ID
observations: List of observation contents to delete
"""
logger.debug(f"Deleting observations from entity {path_id}")
logger.debug(f"Deleting observations from entity {permalink}")
try:
# Get entity
entity = await self.entity_repository.get_by_path_id(path_id)
entity = await self.entity_repository.get_by_permalink(permalink)
if not entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
# Delete observations from DB by comparing the string value to the Observation content
for observation in observations:
@@ -107,7 +107,7 @@ class ObservationService(BaseService[ObservationRepository]):
await self.entity_repository.update(entity.id, {"checksum": checksum})
# Return final entity with all updates
return await self.entity_repository.get_by_path_id(path_id)
return await self.entity_repository.get_by_permalink(permalink)
except Exception as e:
logger.error(f"Failed to delete observations: {e}")
src/basic_memory/services/relation_service.py
+14 -12
View File
@@ -37,8 +37,8 @@ class RelationService(BaseService[RelationRepository]):
for rs in relations:
try:
from_entity = await self.entity_repository.get_by_path_id(rs.from_id)
to_entity = await self.entity_repository.get_by_path_id(rs.to_id)
from_entity = await self.entity_repository.get_by_permalink(rs.from_id)
to_entity = await self.entity_repository.get_by_permalink(rs.to_id)
relation = RelationModel(
from_id=from_entity.id,
@@ -58,10 +58,10 @@ class RelationService(BaseService[RelationRepository]):
continue
# Get fresh copies of all updated entities
for path_id in entities_to_update:
for permalink in entities_to_update:
try:
# Get fresh entity
entity = await self.entity_repository.get_by_path_id(path_id)
entity = await self.entity_repository.get_by_permalink(permalink)
# Write updated file
_, checksum = await self.file_service.write_entity_file(entity)
@@ -70,11 +70,13 @@ class RelationService(BaseService[RelationRepository]):
updated_entities.append(updated)
except Exception as e:
logger.error(f"Failed to update entity {path_id}: {e}")
logger.error(f"Failed to update entity {permalink}: {e}")
continue
# select again to eagerly load all relations
return await self.entity_repository.find_by_path_ids([e.path_id for e in updated_entities])
return await self.entity_repository.find_by_permalinks(
[e.permalink for e in updated_entities]
)
async def delete_relations(self, to_delete: List[RelationSchema]) -> Sequence[EntityModel]:
"""Delete relations and return all updated entities."""
@@ -104,12 +106,12 @@ class RelationService(BaseService[RelationRepository]):
logger.warning("No relations were deleted")
# Get fresh copies of all updated entities
for path_id in entities_to_update:
for permalink in entities_to_update:
try:
# Get fresh entity
entity = await self.entity_repository.get_by_path_id(path_id)
entity = await self.entity_repository.get_by_permalink(permalink)
if not entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
# Write updated file
_, checksum = await self.file_service.write_entity_file(entity)
@@ -118,7 +120,7 @@ class RelationService(BaseService[RelationRepository]):
updated_entities.append(updated)
except Exception as e:
logger.error(f"Failed to update entity {path_id}: {e}")
logger.error(f"Failed to update entity {permalink}: {e}")
continue
return updated_entities
@@ -128,9 +130,9 @@ class RelationService(BaseService[RelationRepository]):
raise
async def find_relation(
self, from_path_id: str, to_path_id: str, relation_type: str
self, from_permalink: str, to_permalink: str, relation_type: str
) -> RelationModel:
return await self.repository.find_relation(from_path_id, to_path_id, relation_type)
return await self.repository.find_relation(from_permalink, to_permalink, relation_type)
async def delete_relation(
self, from_entity: EntityModel, to_entity: EntityModel, relation_type: str
src/basic_memory/services/search_service.py
+7 -7
View File
@@ -60,7 +60,7 @@ class SearchService:
*[f"{obs.category}: {obs.content}" for obs in entity.observations],
# Add relations
*[
f"{rel.relation_type} {rel.to_entity.path_id}: {rel.context or ''}"
f"{rel.relation_type} {rel.to_entity.permalink}: {rel.context or ''}"
for rel in entity.relations
],
]
@@ -77,7 +77,7 @@ class SearchService:
background_tasks.add_task(
self._do_index,
content=content,
path_id=entity.path_id,
permalink=entity.permalink,
file_path=entity.file_path,
type=SearchItemType.ENTITY,
metadata=metadata,
@@ -85,20 +85,20 @@ class SearchService:
else:
await self._do_index(
content=content,
path_id=entity.path_id,
permalink=entity.permalink,
file_path=entity.file_path,
type=SearchItemType.ENTITY,
metadata=metadata,
)
async def _do_index(
self, content: str, path_id: str, file_path: str, type: SearchItemType, metadata: dict
self, content: str, permalink: str, file_path: str, type: SearchItemType, metadata: dict
) -> None:
"""Actually perform the indexing."""
await self.repository.index_item(
content=content, path_id=path_id, file_path=file_path, type=type, metadata=metadata
content=content, permalink=permalink, file_path=file_path, type=type, metadata=metadata
)
async def delete_by_path_id(self, path_id: str):
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
await self.repository.delete_by_path_id(path_id)
await self.repository.delete_by_permalink(permalink)
src/basic_memory/sync/entity_sync_service.py
+11 -11
View File
@@ -25,7 +25,7 @@ def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> Enti
model = EntityModel(
title=markdown.frontmatter.title,
entity_type=markdown.frontmatter.type,
path_id=markdown.frontmatter.id,
permalink=markdown.frontmatter.id,
file_path=file_path,
content_type="text/markdown",
summary=markdown.content.content,
@@ -65,21 +65,21 @@ class EntitySyncService:
model = entity_model_from_markdown(file_path, markdown)
# Mark as incomplete sync
model.checksum = None
model.checksum = None
return await self.entity_repository.add(model)
async def update_entity_and_observations(
self, path_id: str, markdown: EntityMarkdown
self, permalink: str, markdown: EntityMarkdown
) -> EntityModel:
"""First pass: Update entity fields and observations.
Updates everything except relations and sets null checksum
to indicate sync not complete.
"""
logger.debug(f"Updating entity and observations: {path_id}")
db_entity = await self.entity_repository.get_by_path_id(path_id)
logger.debug(f"Updating entity and observations: {permalink}")
db_entity = await self.entity_repository.get_by_permalink(permalink)
if not db_entity:
raise EntityNotFoundError(f"Entity not found: {path_id}")
raise EntityNotFoundError(f"Entity not found: {permalink}")
# Update fields from markdown
db_entity.title = markdown.frontmatter.title
@@ -88,7 +88,7 @@ class EntitySyncService:
# Clear observations for entity
await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
# add new observations
observations = [
Observation(
@@ -122,14 +122,14 @@ class EntitySyncService:
checksum: Final checksum to set after relations are updated
"""
logger.debug(f"Updating relations for entity: {markdown.frontmatter.id}")
db_entity = await self.entity_repository.get_by_path_id(markdown.frontmatter.id)
db_entity = await self.entity_repository.get_by_permalink(markdown.frontmatter.id)
# get all entities from relations
target_entity_path_ids = [rel.target for rel in markdown.content.relations]
target_entities = await self.entity_repository.find_by_path_ids(target_entity_path_ids)
target_entity_permalinks = [rel.target for rel in markdown.content.relations]
target_entities = await self.entity_repository.find_by_permalinks(target_entity_permalinks)
# dict by path
entity_by_path = {e.path_id: e for e in target_entities}
entity_by_path = {e.permalink: e for e in target_entities}
# Clear and update relations
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
src/basic_memory/sync/file_change_scanner.py
+24 -18
View File
@@ -1,7 +1,8 @@
"""Service for detecting changes between filesystem and database."""
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, Sequence, Any
from typing import Dict, Sequence
from loguru import logger
@@ -10,22 +11,24 @@ from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.sync.utils import SyncReport
from basic_memory.utils.file_utils import compute_checksum
@dataclass
class FileState:
"""State of a file including file path, path_id and checksum info."""
"""State of a file including file path, permalink and checksum info."""
file_path: str
path_id: str
permalink: str
checksum: str
@dataclass
@dataclass
class ScanResult:
"""Result of scanning a directory."""
# file_path -> checksum
files: Dict[str, str] = field(default_factory=dict)
# file_path -> error message
errors: Dict[str, str] = field(default_factory=dict)
errors: Dict[str, str] = field(default_factory=dict)
class FileChangeScanner:
@@ -34,9 +37,7 @@ class FileChangeScanner:
The filesystem is treated as the source of truth.
"""
def __init__(
self, entity_repository: EntityRepository
):
def __init__(self, entity_repository: EntityRepository):
self.entity_repository = entity_repository
async def scan_directory(self, directory: Path) -> ScanResult:
@@ -70,7 +71,7 @@ class FileChangeScanner:
checksum = await compute_checksum(content)
if checksum: # Only store valid checksums
result.files[rel_path] = checksum
result.files[rel_path] = checksum
else:
result.errors[rel_path] = "Failed to compute checksum"
@@ -85,13 +86,15 @@ class FileChangeScanner:
return result
async def find_changes(self, directory: Path, db_file_state: Dict[str, FileState]) -> SyncReport:
async def find_changes(
self, directory: Path, db_file_state: Dict[str, FileState]
) -> SyncReport:
"""
Find changes between filesystem and database.
Args:
directory: Directory to check
db_file_state: dict mapping file_path to DbState(path_id, checksum)
db_file_state: dict mapping file_path to DbState(permalink, checksum)
Returns:
SyncReport detailing changes
@@ -102,11 +105,11 @@ class FileChangeScanner:
# Build report
report = SyncReport()
# Find new and modified files
for file_path, checksum in current_files.items():
logger.debug(f"{file_path} ({checksum[:8]})")
if file_path not in db_file_state:
report.new.add(file_path)
elif checksum != db_file_state[file_path].checksum:
@@ -134,9 +137,7 @@ class FileChangeScanner:
return report
async def get_db_file_state(
self, db_records: Sequence[Entity]
) -> Dict[str, FileState]:
async def get_db_file_state(self, db_records: Sequence[Entity]) -> Dict[str, FileState]:
"""Get file_path and checksums from database.
Args:
db_records: database records
@@ -144,7 +145,12 @@ class FileChangeScanner:
Dict mapping file paths to FileState
:param db_records: the data from the db
"""
return {r.file_path: FileState(file_path=r.file_path, path_id=r.path_id, checksum=r.checksum) for r in db_records}
return {
r.file_path: FileState(
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum
)
for r in db_records
}
async def find_knowledge_changes(self, directory: Path) -> SyncReport:
"""Find changes in knowledge directory."""
src/basic_memory/sync/sync_service.py
+3 -3
View File
@@ -56,10 +56,10 @@ class SyncService:
file_path, entity_markdown
)
else:
path_id = entity_markdown.frontmatter.id
logger.debug(f"Updating entity_markdown: {path_id}")
permalink = entity_markdown.frontmatter.id
logger.debug(f"Updating entity_markdown: {permalink}")
await self.knowledge_sync_service.update_entity_and_observations(
path_id, entity_markdown
permalink, entity_markdown
)
# Second pass: Process relations