feat: fast edit entities, refactors for webui, enhance search (#532)

Signed-off-by: phernandez <paul@basicmachines.co>

src/basic_memory/services/entity_service.py

@@ -1,5 +1,6 @@
 """Service for managing entities in the database."""
 
+from datetime import datetime
 from pathlib import Path
 from typing import List, Optional, Sequence, Tuple, Union
 
@@ -17,7 +18,7 @@ from basic_memory.file_utils import (
     dump_frontmatter,
 )
 from basic_memory.markdown import EntityMarkdown
-from basic_memory.markdown.entity_parser import EntityParser
+from basic_memory.markdown.entity_parser import EntityParser, normalize_frontmatter_metadata
 from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown
 from basic_memory.models import Entity as EntityModel
 from basic_memory.models import Observation, Relation
@@ -167,6 +168,25 @@ class EntityService(BaseService[EntityModel]):
 
         return permalink
 
+    def _build_frontmatter_markdown(
+        self, title: str, entity_type: str, permalink: str
+    ) -> EntityMarkdown:
+        """Build a minimal EntityMarkdown object for permalink resolution."""
+        from basic_memory.markdown.schemas import EntityFrontmatter
+
+        frontmatter_metadata = {
+            "title": title,
+            "type": entity_type,
+            "permalink": permalink,
+        }
+        frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata)
+        return EntityMarkdown(
+            frontmatter=frontmatter_obj,
+            content="",
+            observations=[],
+            relations=[],
+        )
+
     async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
         """Create new entity or update existing one.
         Returns: (entity, is_new) where is_new is True if a new entity was created
@@ -210,20 +230,8 @@ class EntityService(BaseService[EntityModel]):
                 schema.entity_type = content_frontmatter["type"]
 
             if "permalink" in content_frontmatter:
-                # Create a minimal EntityMarkdown object for permalink resolution
-                from basic_memory.markdown.schemas import EntityFrontmatter
-
-                frontmatter_metadata = {
-                    "title": schema.title,
-                    "type": schema.entity_type,
-                    "permalink": content_frontmatter["permalink"],
-                }
-                frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata)
-                content_markdown = EntityMarkdown(
-                    frontmatter=frontmatter_obj,
-                    content="",  # content not needed for permalink resolution
-                    observations=[],
-                    relations=[],
+                content_markdown = self._build_frontmatter_markdown(
+                    schema.title, schema.entity_type, content_frontmatter["permalink"]
                 )
 
         # Get unique permalink (prioritizing content frontmatter) unless disabled
@@ -248,11 +256,8 @@ class EntityService(BaseService[EntityModel]):
             content=final_content,
         )
 
-        # create entity
-        created = await self.create_entity_from_markdown(file_path, entity_markdown)
-
-        # add relations
-        entity = await self.update_entity_relations(created.file_path, entity_markdown)
+        # create entity and relations
+        entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=True)
 
         # Set final checksum to mark complete
         return await self.repository.update(entity.id, {"checksum": checksum})
@@ -283,20 +288,8 @@ class EntityService(BaseService[EntityModel]):
                 schema.entity_type = content_frontmatter["type"]
 
             if "permalink" in content_frontmatter:
-                # Create a minimal EntityMarkdown object for permalink resolution
-                from basic_memory.markdown.schemas import EntityFrontmatter
-
-                frontmatter_metadata = {
-                    "title": schema.title,
-                    "type": schema.entity_type,
-                    "permalink": content_frontmatter["permalink"],
-                }
-                frontmatter_obj = EntityFrontmatter(metadata=frontmatter_metadata)
-                content_markdown = EntityMarkdown(
-                    frontmatter=frontmatter_obj,
-                    content="",  # content not needed for permalink resolution
-                    observations=[],
-                    relations=[],
+                content_markdown = self._build_frontmatter_markdown(
+                    schema.title, schema.entity_type, content_frontmatter["permalink"]
                 )
 
         # Check if we need to update the permalink based on content frontmatter (unless disabled)
@@ -333,17 +326,179 @@ class EntityService(BaseService[EntityModel]):
             content=final_content,
         )
 
-        # update entity in db
-        entity = await self.update_entity_and_observations(file_path, entity_markdown)
-
-        # add relations
-        await self.update_entity_relations(file_path.as_posix(), entity_markdown)
+        # update entity and relations
+        entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
 
         # Set final checksum to match file
         entity = await self.repository.update(entity.id, {"checksum": checksum})
 
         return entity
 
+    async def fast_write_entity(
+        self,
+        schema: EntitySchema,
+        external_id: Optional[str] = None,
+    ) -> EntityModel:
+        """Write file and upsert a minimal entity row for fast responses."""
+        logger.debug(
+            "Fast-writing entity",
+            title=schema.title,
+            external_id=external_id,
+            content_type=schema.content_type,
+        )
+
+        # --- Identity & File Path ---
+        existing = await self.repository.get_by_external_id(external_id) if external_id else None
+
+        # Trigger: external_id already exists
+        # Why: avoid duplicate entities when title-derived paths change
+        # Outcome: update in-place and keep the existing file path
+        file_path = Path(existing.file_path) if existing else Path(schema.file_path)
+
+        if not existing and await self.file_service.exists(file_path):
+            raise EntityCreationError(
+                f"file for entity {schema.directory}/{schema.title} already exists: {file_path}"
+            )
+
+        # --- Frontmatter Overrides ---
+        content_markdown = None
+        if schema.content and has_frontmatter(schema.content):
+            content_frontmatter = parse_frontmatter(schema.content)
+
+            if "type" in content_frontmatter:
+                schema.entity_type = content_frontmatter["type"]
+
+            if "permalink" in content_frontmatter:
+                content_markdown = self._build_frontmatter_markdown(
+                    schema.title, schema.entity_type, content_frontmatter["permalink"]
+                )
+
+        # --- Permalink Resolution ---
+        if self.app_config and self.app_config.disable_permalinks:
+            schema._permalink = ""
+        else:
+            if existing and not (content_markdown and content_markdown.frontmatter.permalink):
+                schema._permalink = existing.permalink or await self.resolve_permalink(
+                    file_path, skip_conflict_check=True
+                )
+            else:
+                schema._permalink = await self.resolve_permalink(
+                    file_path, content_markdown, skip_conflict_check=True
+                )
+
+        # --- File Write ---
+        post = await schema_to_markdown(schema)
+        final_content = dump_frontmatter(post)
+        checksum = await self.file_service.write_file(file_path, final_content)
+
+        # --- Minimal DB Upsert ---
+        metadata = normalize_frontmatter_metadata(post.metadata or {})
+        entity_metadata = {k: v for k, v in metadata.items() if v is not None}
+        update_data = {
+            "title": schema.title,
+            "entity_type": schema.entity_type,
+            "file_path": file_path.as_posix(),
+            "content_type": schema.content_type,
+            "entity_metadata": entity_metadata or None,
+            "permalink": schema.permalink,
+            "checksum": checksum,
+            "updated_at": datetime.now().astimezone(),
+        }
+
+        if existing:
+            updated = await self.repository.update(existing.id, update_data)
+            if not updated:
+                raise ValueError(f"Failed to update entity in database: {existing.id}")
+            return updated
+
+        create_data = {
+            **update_data,
+            "external_id": external_id,
+        }
+        return await self.repository.create(create_data)
+
+    async def fast_edit_entity(
+        self,
+        entity: EntityModel,
+        operation: str,
+        content: str,
+        section: Optional[str] = None,
+        find_text: Optional[str] = None,
+        expected_replacements: int = 1,
+    ) -> EntityModel:
+        """Edit an entity quickly and defer full indexing to background."""
+        logger.debug(f"Fast editing entity: {entity.external_id}, operation: {operation}")
+
+        # --- File Edit ---
+        file_path = Path(entity.file_path)
+        current_content, _ = await self.file_service.read_file(file_path)
+        new_content = self.apply_edit_operation(
+            current_content, operation, content, section, find_text, expected_replacements
+        )
+        checksum = await self.file_service.write_file(file_path, new_content)
+
+        # --- Frontmatter Overrides ---
+        update_data = {
+            "checksum": checksum,
+            "updated_at": datetime.now().astimezone(),
+        }
+        content_markdown = None
+        if has_frontmatter(new_content):
+            content_frontmatter = parse_frontmatter(new_content)
+
+            if "title" in content_frontmatter:
+                update_data["title"] = content_frontmatter["title"]
+            if "type" in content_frontmatter:
+                update_data["entity_type"] = content_frontmatter["type"]
+
+            if "permalink" in content_frontmatter:
+                content_markdown = self._build_frontmatter_markdown(
+                    update_data.get("title", entity.title),
+                    update_data.get("entity_type", entity.entity_type),
+                    content_frontmatter["permalink"],
+                )
+
+            metadata = normalize_frontmatter_metadata(content_frontmatter or {})
+            update_data["entity_metadata"] = {k: v for k, v in metadata.items() if v is not None}
+
+        # --- Permalink Resolution ---
+        if self.app_config and self.app_config.disable_permalinks:
+            update_data["permalink"] = None
+        elif content_markdown and content_markdown.frontmatter.permalink:
+            update_data["permalink"] = await self.resolve_permalink(
+                file_path, content_markdown, skip_conflict_check=True
+            )
+
+        updated = await self.repository.update(entity.id, update_data)
+        if not updated:
+            raise ValueError(f"Failed to update entity in database: {entity.id}")
+        return updated
+
+    async def reindex_entity(self, entity_id: int) -> None:
+        """Parse file content and rebuild observations/relations/search for an entity."""
+        entity = await self.repository.find_by_id(entity_id)
+        if not entity:
+            raise EntityNotFoundError(f"Entity not found: {entity_id}")
+
+        # --- Full Parse ---
+        file_path = Path(entity.file_path)
+        content = await self.file_service.read_file_content(file_path)
+        entity_markdown = await self.entity_parser.parse_markdown_content(
+            file_path=file_path,
+            content=content,
+        )
+
+        # --- DB Reindex ---
+        updated = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
+        checksum = await self.file_service.compute_checksum(file_path)
+        updated = await self.repository.update(updated.id, {"checksum": checksum})
+        if not updated:
+            raise ValueError(f"Failed to update entity in database: {entity.id}")
+
+        # --- Search Reindex ---
+        if self.search_service:
+            await self.search_service.index_entity_data(updated, content=content)
+
     async def delete_entity(self, permalink_or_id: str | int) -> bool:
         """Delete entity and its file."""
         logger.debug(f"Deleting entity: {permalink_or_id}")
@@ -465,6 +620,20 @@ class EntityService(BaseService[EntityModel]):
             db_entity,
         )
 
+    async def upsert_entity_from_markdown(
+        self,
+        file_path: Path,
+        markdown: EntityMarkdown,
+        *,
+        is_new: bool,
+    ) -> EntityModel:
+        """Create/update entity and relations from parsed markdown."""
+        if is_new:
+            created = await self.create_entity_from_markdown(file_path, markdown)
+        else:
+            created = await self.update_entity_and_observations(file_path, markdown)
+        return await self.update_entity_relations(created.file_path, markdown)
+
     async def update_entity_relations(
         self,
         path: str,
@@ -589,8 +758,7 @@ class EntityService(BaseService[EntityModel]):
         )
 
         # Update entity and its relationships
-        entity = await self.update_entity_and_observations(file_path, entity_markdown)
-        await self.update_entity_relations(file_path.as_posix(), entity_markdown)
+        entity = await self.upsert_entity_from_markdown(file_path, entity_markdown, is_new=False)
 
         # Set final checksum to match file
         entity = await self.repository.update(entity.id, {"checksum": checksum})
@@ -927,7 +1095,9 @@ class EntityService(BaseService[EntityModel]):
                 old_path = entity.file_path
                 # Replace only the first occurrence of the source directory prefix
                 if old_path.startswith(f"{source_directory}/"):
-                    new_path = old_path.replace(f"{source_directory}/", f"{destination_directory}/", 1)
+                    new_path = old_path.replace(
+                        f"{source_directory}/", f"{destination_directory}/", 1
+                    )
                 else:  # pragma: no cover
                     # Entity is directly in the source directory (shouldn't happen with prefix match)
                     new_path = f"{destination_directory}/{old_path}"
@@ -1016,7 +1186,9 @@ class EntityService(BaseService[EntityModel]):
                     logger.debug(f"Deleted entity: {file_path}")
                 else:  # pragma: no cover
                     failed_deletes += 1
-                    errors.append(DirectoryDeleteError(path=file_path, error="Delete returned False"))
+                    errors.append(
+                        DirectoryDeleteError(path=file_path, error="Delete returned False")
+                    )
                     logger.warning(f"Delete returned False for entity: {file_path}")
 
             except Exception as e:  # pragma: no cover

src/basic_memory/services/search_service.py

@@ -2,7 +2,7 @@
 
 import ast
 from datetime import datetime
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Dict, Any
 
 
 from dateparser import parse
@@ -95,6 +95,15 @@ class SearchService:
             else None
         )
 
+        # Merge structured metadata filters (explicit + convenience fields)
+        metadata_filters: Optional[Dict[str, Any]] = None
+        if query.metadata_filters or query.tags or query.status:
+            metadata_filters = dict(query.metadata_filters or {})
+            if query.tags:
+                metadata_filters.setdefault("tags", query.tags)
+            if query.status:
+                metadata_filters.setdefault("status", query.status)
+
         # search
         results = await self.repository.search(
             search_text=query.text,
@@ -104,6 +113,7 @@ class SearchService:
             types=query.types,
             search_item_types=query.entity_types,
             after_date=after_date,
+            metadata_filters=metadata_filters,
             limit=limit,
             offset=offset,
         )