add category to observation schema

src/basic_memory/api/routers/__init__.py

@@ -1,5 +1,6 @@
 """API routers."""
 
-from . import knowledge
+from . import knowledge_router as knowledge
+from . import documents_router as documents
 
-__all__ = ["knowledge"]
+__all__ = ["knowledge", "documents"]

src/basic_memory/models/documents.py

@@ -3,7 +3,7 @@
 from datetime import datetime
 from typing import Optional, List
 
-from sqlalchemy import String, DateTime, text, JSON
+from sqlalchemy import String, DateTime, text, JSON, Index
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from basic_memory.models.base import Base
@@ -19,6 +19,10 @@ class Document(Base):
     """
 
     __tablename__ = "document"
+    __table_args__ = (
+        Index("ix_document_created_at", "created_at"),
+        Index("ix_document_updated_at", "updated_at")
+    )
 
     id: Mapped[int] = mapped_column(primary_key=True)
     path: Mapped[str] = mapped_column(String, unique=True, nullable=False, index=True)

src/basic_memory/models/knowledge.py

@@ -8,6 +8,7 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from basic_memory.models.base import Base
 from basic_memory.models.documents import Document
+from enum import Enum
 
 
 class Entity(Base):
@@ -26,6 +27,8 @@ class Entity(Base):
         UniqueConstraint("entity_type", "name", name="uix_entity_type_name"),
         Index("ix_entity_type", "entity_type"),
         Index("ix_entity_doc_id", "doc_id"),
+        Index("ix_entity_created_at", "created_at"),  # For timeline queries
+        Index("ix_entity_updated_at", "updated_at")   # For timeline queries
     )
 
     # Core identity
@@ -75,6 +78,14 @@ class Entity(Base):
         return f"Entity(id={self.id}, name='{self.name}', type='{self.entity_type}')"
 
 
+class ObservationCategory(str, Enum):
+    TECH = "tech"
+    DESIGN = "design"
+    FEATURE = "feature"
+    NOTE = "note"
+    ISSUE = "issue"
+    TODO = "todo"
+
 class Observation(Base):
     """
     An observation about an entity.
@@ -83,10 +94,22 @@ class Observation(Base):
     """
 
     __tablename__ = "observation"
+    __table_args__ = (
+        Index("ix_observation_entity_id", "entity_id"), # Add FK index
+        Index("ix_observation_category", "category"),    # Add category index
+        Index("ix_observation_created_at", "created_at"),  # For timeline queries
+        Index("ix_observation_updated_at", "updated_at"),  # For timeline queries
+    )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)
     entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
     content: Mapped[str] = mapped_column(Text)
+    category: Mapped[str] = mapped_column(
+        String,
+        nullable=False,
+        default=ObservationCategory.NOTE.value,
+        server_default=ObservationCategory.NOTE.value
+    )
     context: Mapped[str] = mapped_column(Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
     updated_at: Mapped[datetime] = mapped_column(
@@ -109,6 +132,10 @@ class Relation(Base):
     __table_args__ = (
         UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
         Index("ix_relation_type", "relation_type"),
+        Index("ix_relation_from_id", "from_id"),  # Add FK indexes
+        Index("ix_relation_to_id", "to_id"),
+        Index("ix_relation_created_at", "created_at"),  # For timeline queries
+        Index("ix_relation_updated_at", "updated_at"),  # For timeline queries
     )
 
     id: Mapped[int] = mapped_column(Integer, primary_key=True)

src/basic_memory/schemas/base.py

@@ -33,6 +33,7 @@ Common Relation Types:
 """
 
 import re
+from enum import Enum
 from typing import List, Optional, Annotated
 
 from annotated_types import MinLen, MaxLen
@@ -67,6 +68,15 @@ def validate_path_format(path: str) -> str:
 
     return path
 
+
+class ObservationCategory(str, Enum):
+    TECH = "tech"
+    DESIGN = "design"
+    FEATURE = "feature"
+    NOTE = "note"
+    ISSUE = "issue"
+    TODO = "todo"
+
 PathId = Annotated[str, BeforeValidator(to_snake_case), BeforeValidator(validate_path_format)]
 """Unique identifier in format '{path}/{normalized_name}'."""
 

src/basic_memory/schemas/request.py

@@ -2,11 +2,17 @@
 
 from typing import List, Optional, Annotated, Dict, Any
 from annotated_types import MaxLen, MinLen
-from pydantic.json_schema import Pattern
 
 from pydantic import BaseModel, StringConstraints
 
-from basic_memory.schemas.base import Observation, Entity, Relation, PathId
+from basic_memory.schemas.base import Observation, Entity, Relation, PathId, ObservationCategory
+
+
+class ObservationCreate(BaseModel):
+    """A single observation with category, content, and optional context."""
+
+    category: ObservationCategory = ObservationCategory.NOTE
+    content: Observation 
 
 
 class AddObservationsRequest(BaseModel):
@@ -22,21 +28,31 @@ class AddObservationsRequest(BaseModel):
     {
         "path_id": "component/memory_service",
         "observations": [
-            "Added support for async operations",
-            "Improved error handling with custom exceptions",
-            "Now uses SQLAlchemy 2.0 features"
-        ]
+            {
+                "category": "feature",
+                "content": "Added support for async operations",
+            },
+            {
+                "category": "tech",
+                "content": "Improved error handling with custom exceptions",
+            }
+        ]   
     }
 
     2. Documenting a decision:
     {
         "path_id": "decision/db_schema_design",
-        "observations": [
-            "Chose SQLite for local-first storage",
-            "Added support for full-text search via FTS5",
-            "Implemented proper foreign key constraints"
-        ],
         "context": "Initial database design meeting"
+        "observations": [
+            {
+                "category": "feature",
+                "content": "Added support for async operations",
+            },
+            {
+                "category": "tech",
+                "content": "Improved error handling with custom exceptions",
+            }
+        ]
     }
 
     Best Practices:
@@ -48,7 +64,7 @@ class AddObservationsRequest(BaseModel):
 
     path_id: PathId
     context: Optional[str] = None
-    observations: List[Observation]
+    observations: List[ObservationCreate]
 
 
 class CreateEntityRequest(BaseModel):
@@ -57,6 +73,8 @@ class CreateEntityRequest(BaseModel):
     Entities represent nodes in the knowledge graph. They can be created
     with initial observations and optional descriptions. Entity IDs are
     automatically generated from the type and name.
+    
+    Observations will be assigned the default category of 'note'.
 
     Example Request:
     {
@@ -203,10 +221,7 @@ class CreateRelationsRequest(BaseModel):
 
 
 FilePath = Annotated[
-    str,
-    StringConstraints(pattern=r'^[a-zA-Z0-9_/.-]+\.md$'),
-    MinLen(1),
-    MaxLen(255)
+    str, StringConstraints(pattern=r"^[a-zA-Z0-9_/.-]+\.md$"), MinLen(1), MaxLen(255)
 ]
 
 

src/basic_memory/schemas/response.py

@@ -17,6 +17,7 @@ from typing import List, Optional, Dict, Any
 from pydantic import BaseModel, ConfigDict, Field, AliasPath, AliasChoices
 
 from basic_memory.schemas.base import Observation, Relation, PathId, Entity, EntityType
+from basic_memory.schemas.request import ObservationCreate
 
 
 class SQLAlchemyModel(BaseModel):
@@ -30,7 +31,7 @@ class SQLAlchemyModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
-class ObservationResponse(SQLAlchemyModel):
+class ObservationResponse(ObservationCreate, SQLAlchemyModel):
     """Schema for observation data returned from the service.
 
     Each observation gets a unique ID that can be used for later
@@ -38,11 +39,12 @@ class ObservationResponse(SQLAlchemyModel):
 
     Example Response:
     {
-        "content": "Implements SQLite storage for persistence"
+        "category": "feature",
+        "content": "Added support for async operations",
+        "context": "Initial database design meeting"
     }
     """
-
-    content: Observation
+    context: Optional[str] = None
 
 
 class RelationResponse(Relation, SQLAlchemyModel):
@@ -94,10 +96,14 @@ class EntityResponse(SQLAlchemyModel):
         "description": "Core persistence service",
         "observations": [
             {
+                "category": "feature",
                 "content": "Uses SQLite storage"
+                "context": "Initial design"
             },
             {
+                "category": "feature",
                 "content": "Implements async operations"
+                "context": "Initial design"
             }
         ],
         "relations": [