sanitize inputs for id/name

This commit is contained in:
phernandez
2024-12-16 19:35:18 -06:00
parent 176ebb3d55
commit 86329e0105
16 changed files with 779 additions and 416 deletions
+3 -4
View File
@@ -112,13 +112,12 @@ async def delete_entity(
return DeleteEntityResponse(deleted=deleted)
@router.post(
"/entities/{entity_id:path}/observations/delete", response_model=DeleteObservationsResponse
)
@router.post("/observations/delete", response_model=DeleteObservationsResponse)
async def delete_observations(
entity_id: str, data: DeleteObservationsRequest, memory_service: MemoryServiceDep
data: DeleteObservationsRequest, memory_service: MemoryServiceDep
) -> DeleteObservationsResponse:
"""Delete observations from an entity."""
entity_id = data.entity_id
deleted = await memory_service.delete_observations(entity_id, data.deletions)
return DeleteObservationsResponse(deleted=deleted)
+5 -3
View File
@@ -160,9 +160,11 @@ class MemoryServer(Server):
"open_nodes": lambda c, a: c.post("/knowledge/nodes", json=a),
"add_observations": lambda c, a: c.post("/knowledge/observations", json=a),
"create_relations": lambda c, a: c.post("/knowledge/relations", json=a),
"delete_entities": lambda c, a: c.post(f"/knowledge/entities/{a['names'][0]}"),
"delete_observations": lambda c, a: c.post("/knowledge/observations", json=a),
"delete_relations": lambda c, a: c.post("/knowledge/entities/", json=a),
"delete_entities": lambda c, a: c.post("/knowledge/entities/delete", json=a),
"delete_observations": lambda c, a: c.post(
"/knowledge/observations/delete", json=a
),
"delete_relations": lambda c, a: c.post("/knowledge/relations/delete", json=a),
}
# Get handler for tool
+26 -49
View File
@@ -1,15 +1,18 @@
"""Database models for basic-memory."""
from datetime import datetime
from typing import List, Optional
from sqlalchemy import String, DateTime, ForeignKey, Text, Integer, text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase
from sqlalchemy.ext.asyncio import AsyncAttrs
from basic_memory.utils import normalize_entity_id
from sqlalchemy import String, DateTime, ForeignKey, Text, Integer, text, UniqueConstraint
from sqlalchemy.ext.asyncio import AsyncAttrs
from sqlalchemy.orm import Mapped, mapped_column, relationship, DeclarativeBase
from basic_memory.utils import sanitize_name
class Base(AsyncAttrs, DeclarativeBase):
"""Base class for all models"""
pass
@@ -24,54 +27,45 @@ class Entity(Base):
- A description (optional)
- A list of observations
"""
__tablename__ = "entity"
__table_args__ = (
UniqueConstraint('entity_type', 'name', name='uix_entity_type_name'),
)
__table_args__ = (UniqueConstraint("entity_type", "name", name="uix_entity_type_name"),)
id: Mapped[str] = mapped_column(String, primary_key=True)
name: Mapped[str] = mapped_column(String)
entity_type: Mapped[str] = mapped_column(String)
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime,
server_default=text('CURRENT_TIMESTAMP')
)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
updated_at: Mapped[datetime] = mapped_column(
DateTime,
server_default=text('CURRENT_TIMESTAMP'),
onupdate=text('CURRENT_TIMESTAMP')
DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP")
)
# Relationships
observations: Mapped[List["Observation"]] = relationship(
"Observation",
back_populates="entity",
cascade="all, delete-orphan"
"Observation", back_populates="entity", cascade="all, delete-orphan"
)
outgoing_relations: Mapped[List["Relation"]] = relationship(
"Relation",
foreign_keys="[Relation.from_id]",
back_populates="from_entity",
cascade="all, delete-orphan"
cascade="all, delete-orphan",
)
incoming_relations: Mapped[List["Relation"]] = relationship(
"Relation",
foreign_keys="[Relation.to_id]",
back_populates="to_entity",
cascade="all, delete-orphan"
cascade="all, delete-orphan",
)
@property
def relations(self):
return self.outgoing_relations + self.incoming_relations
@classmethod
def generate_id(cls, entity_type: str, name: str) -> str:
"""Generate a filesystem path-based ID for this entity."""
# Use common normalization for filesystem safety
safe_name = normalize_entity_id(name)
safe_name = sanitize_name(name)
return f"{entity_type}/{safe_name}"
def get_file_path(self) -> str:
@@ -90,26 +84,19 @@ class Observation(Base):
- Can be added or removed independently
- Should be atomic (one fact per observation)
"""
__tablename__ = "observation"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
entity_id: Mapped[str] = mapped_column( # Reference Entity.id which is text
String,
ForeignKey("entity.id", ondelete="CASCADE"),
index=True
String, ForeignKey("entity.id", ondelete="CASCADE"), index=True
)
content: Mapped[str] = mapped_column(String)
created_at: Mapped[datetime] = mapped_column(
DateTime,
server_default=text('CURRENT_TIMESTAMP')
)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
context: Mapped[str | None] = mapped_column(String, nullable=True)
# Relationships
entity: Mapped[Entity] = relationship(
"Entity",
back_populates="observations"
)
entity: Mapped[Entity] = relationship("Entity", back_populates="observations")
def __repr__(self) -> str:
content = self.content[:50] + "..." if len(self.content) > 50 else self.content
@@ -121,37 +108,27 @@ class Relation(Base):
Relations define directed connections between entities.
They are always stored in active voice and describe how entities interact or relate to each other.
"""
__tablename__ = "relation"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
from_id: Mapped[str] = mapped_column( # Reference Entity.id which is text
String,
ForeignKey("entity.id", ondelete="CASCADE"),
index=True
String, ForeignKey("entity.id", ondelete="CASCADE"), index=True
)
to_id: Mapped[str] = mapped_column( # Reference Entity.id which is text
String,
ForeignKey("entity.id", ondelete="CASCADE"),
index=True
String, ForeignKey("entity.id", ondelete="CASCADE"), index=True
)
relation_type: Mapped[str] = mapped_column(String)
created_at: Mapped[datetime] = mapped_column(
DateTime,
server_default=text('CURRENT_TIMESTAMP')
)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
context: Mapped[str | None] = mapped_column(String, nullable=True)
# Relationships
from_entity: Mapped[Entity] = relationship(
"Entity",
foreign_keys=[from_id],
back_populates="outgoing_relations"
"Entity", foreign_keys=[from_id], back_populates="outgoing_relations"
)
to_entity: Mapped[Entity] = relationship(
"Entity",
foreign_keys=[to_id],
back_populates="incoming_relations"
"Entity", foreign_keys=[to_id], back_populates="incoming_relations"
)
def __repr__(self) -> str:
return f"Relation(id={self.id}, from='{self.from_id}', type='{self.relation_type}', to='{self.to_id}')"
return f"Relation(id={self.id}, from='{self.from_id}', type='{self.relation_type}', to='{self.to_id}')"
+23 -14
View File
@@ -1,17 +1,26 @@
"""Core pydantic models for basic-memory entities, observations, and relations."""
from typing import List, Optional, Annotated, TypeAlias
from typing import List, Optional, Annotated
from annotated_types import Len
from annotated_types import MinLen, MaxLen
from pydantic import BaseModel, ConfigDict, BeforeValidator
from basic_memory.utils import normalize_entity_id
from basic_memory.utils import sanitize_name
# Base Models
Observation: TypeAlias = str
# Strip whitespace
def strip_whitespace(obs: str) -> str:
return obs.strip()
Observation = Annotated[str, BeforeValidator(strip_whitespace), MinLen(1), MaxLen(1000)]
EntityType = Annotated[str, BeforeValidator(strip_whitespace), MinLen(1), MaxLen(20)]
RelationType = Annotated[str, BeforeValidator(strip_whitespace), MinLen(1), MaxLen(20)]
# Custom field types with validation
EntityId = Annotated[str, BeforeValidator(normalize_entity_id)]
EntityId = Annotated[str, BeforeValidator(sanitize_name)]
class Relation(BaseModel):
@@ -22,7 +31,7 @@ class Relation(BaseModel):
from_id: EntityId
to_id: EntityId
relation_type: str
relation_type: RelationType
context: Optional[str] = None
@@ -35,7 +44,7 @@ class Entity(BaseModel):
id: Optional[EntityId] = None
name: str
entity_type: str
entity_type: EntityType
description: Optional[str] = None
observations: List[Observation] = []
relations: List[Relation] = []
@@ -60,19 +69,19 @@ class AddObservationsRequest(BaseModel):
class CreateEntityRequest(BaseModel):
"""Request schema for create_entities tool."""
entities: Annotated[List[Entity], Len(min_length=1)]
entities: Annotated[List[Entity], MinLen(1)]
class SearchNodesRequest(BaseModel):
"""Request schema for search_nodes tool."""
query: str
query: Annotated[str, MinLen(1), MaxLen(200)]
class OpenNodesRequest(BaseModel):
"""Request schema for open_nodes tool."""
names: Annotated[List[EntityId], Len(min_length=1)]
names: Annotated[List[EntityId], MinLen(1)]
class CreateRelationsRequest(BaseModel):
@@ -87,7 +96,7 @@ class CreateRelationsRequest(BaseModel):
class DeleteEntityRequest(BaseModel):
"""Request schema for delete_entities tool."""
entity_ids: List[EntityId]
entity_ids: Annotated[List[EntityId], MinLen(1)]
class DeleteRelationsRequest(BaseModel):
@@ -100,7 +109,7 @@ class DeleteObservationsRequest(BaseModel):
"""Request schema for delete_observations tool."""
entity_id: EntityId
deletions: List[Observation]
deletions: Annotated[List[Observation], MinLen(1)]
# response output models
@@ -134,7 +143,7 @@ class RelationResponse(Relation, SQLAlchemyModel):
class EntityResponse(SQLAlchemyModel):
"""Schema for entity data returned from the service."""
id: EntityId
id: str
name: str
entity_type: str
description: Optional[str] = None
+25 -9
View File
@@ -1,13 +1,29 @@
"""Utility functions for basic-memory."""
def normalize_entity_id(entity_id: str) -> str:
import re
import unicodedata
def sanitize_name(name: str) -> str:
"""
Normalize an entity ID by converting to lowercase and replacing spaces with underscores.
Args:
entity_id: Raw entity ID to normalize
Returns:
Normalized entity ID suitable for filesystem and database use
Sanitize a name for filesystem use:
- Convert to lowercase
- Replace spaces/punctuation with underscores
- Remove emojis and other special characters
- Collapse multiple underscores
- Trim leading/trailing underscores
"""
return entity_id.lower().replace(" ", "_")
# Normalize unicode to compose characters where possible
name = unicodedata.normalize("NFKD", name)
# Remove emojis and other special characters, keep only letters, numbers, spaces
name = "".join(c for c in name if c.isalnum() or c.isspace())
# Replace spaces with underscores
name = name.replace(" ", "_")
# Remove newline
name = name.replace("\n", "")
# Convert to lowercase
name = name.lower()
# Collapse multiple underscores and trim
name = re.sub(r"_+", "_", name).strip("_")
return name