remove logfire instrumentation

Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
Paul Hernandez
2025-12-10 22:17:56 -06:00
committed by phernandez
parent 70bb10be1d
commit 4a43d7df4a
43 changed files with 560 additions and 1181 deletions
+1 -7
View File
@@ -22,12 +22,11 @@ from basic_memory.markdown.schemas import (
Relation,
)
from basic_memory.utils import parse_tags
import logfire
md = MarkdownIt().use(observation_plugin).use(relation_plugin)
@logfire.instrument()
def normalize_frontmatter_value(value: Any) -> Any:
"""Normalize frontmatter values to safe types for processing.
@@ -89,7 +88,6 @@ def normalize_frontmatter_value(value: Any) -> Any:
return value
@logfire.instrument()
def normalize_frontmatter_metadata(metadata: dict) -> dict:
"""Normalize all values in frontmatter metadata dict.
@@ -112,7 +110,6 @@ class EntityContent:
relations: list[Relation] = field(default_factory=list)
@logfire.instrument()
def parse(content: str) -> EntityContent:
"""Parse markdown content into EntityMarkdown."""
@@ -171,7 +168,6 @@ class EntityParser:
return parsed
return None
@logfire.instrument()
async def parse_file(self, path: Path | str) -> EntityMarkdown:
"""Parse markdown file into EntityMarkdown."""
@@ -193,7 +189,6 @@ class EntityParser:
"""Get absolute path for a file using the base path for the project."""
return self.base_path / path
@logfire.instrument()
async def parse_file_content(self, absolute_path, file_content):
"""Parse markdown content from file stats.
@@ -211,7 +206,6 @@ class EntityParser:
ctime=file_stats.st_ctime,
)
@logfire.instrument()
async def parse_markdown_content(
self,
file_path: Path,
@@ -4,7 +4,7 @@ from collections import OrderedDict
from frontmatter import Post
from loguru import logger
import logfire
from basic_memory import file_utils
from basic_memory.file_utils import dump_frontmatter
@@ -40,7 +40,6 @@ class MarkdownProcessor:
"""Initialize processor with base path and parser."""
self.entity_parser = entity_parser
@logfire.instrument()
async def read_file(self, path: Path) -> EntityMarkdown:
"""Read and parse file into EntityMarkdown schema.
@@ -49,7 +48,6 @@ class MarkdownProcessor:
"""
return await self.entity_parser.parse_file(path)
@logfire.instrument()
async def write_file(
self,
path: Path,
@@ -127,7 +125,6 @@ class MarkdownProcessor:
await file_utils.write_file_atomic(path, final_content)
return await file_utils.compute_checksum(final_content)
@logfire.instrument()
def format_observations(self, observations: list[Observation]) -> str:
"""Format observations section in standard way.
@@ -136,7 +133,6 @@ class MarkdownProcessor:
lines = [f"{obs}" for obs in observations]
return "\n".join(lines) + "\n"
@logfire.instrument()
def format_relations(self, relations: list[Relation]) -> str:
"""Format relations section in standard way.
+1 -3
View File
@@ -2,7 +2,7 @@
from pathlib import Path
from typing import Any, Optional
import logfire
from frontmatter import Post
@@ -12,7 +12,6 @@ from basic_memory.models import Entity
from basic_memory.models import Observation as ObservationModel
@logfire.instrument()
def entity_model_from_markdown(
file_path: Path,
markdown: EntityMarkdown,
@@ -74,7 +73,6 @@ def entity_model_from_markdown(
return model
@logfire.instrument()
async def schema_to_markdown(schema: Any) -> Post:
"""
Convert schema to markdown Post object.
-2
View File
@@ -4,7 +4,6 @@ import basic_memory
from basic_memory.models.base import Base
from basic_memory.models.knowledge import Entity, Observation, Relation
from basic_memory.models.project import Project
from basic_memory.models.search import SearchIndex
__all__ = [
"Base",
@@ -12,6 +11,5 @@ __all__ = [
"Observation",
"Relation",
"Project",
"SearchIndex",
"basic_memory",
]
+43 -46
View File
@@ -1,55 +1,52 @@
"""Search models and tables."""
"""Search DDL statements for SQLite and Postgres.
from sqlalchemy import DDL, Column, Integer, String, DateTime, Text, ForeignKey
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.types import JSON
The search_index table is created via raw DDL, not ORM models, because:
- SQLite uses FTS5 virtual tables (cannot be represented as ORM)
- Postgres uses composite primary keys and generated tsvector columns
- Both backends use raw SQL for all search operations via SearchIndexRow dataclass
"""
from basic_memory.models.base import Base
from sqlalchemy import DDL
class SearchIndex(Base):
"""Search index table for Postgres only.
# Define Postgres search_index table with composite primary key and tsvector
# This DDL matches the Alembic migration schema (314f1ea54dc4)
# Used by tests to create the table without running full migrations
# NOTE: Split into separate DDL statements because asyncpg doesn't support
# multiple statements in a single execute call.
CREATE_POSTGRES_SEARCH_INDEX_TABLE = DDL("""
CREATE TABLE IF NOT EXISTS search_index (
id INTEGER NOT NULL,
project_id INTEGER NOT NULL,
title TEXT,
content_stems TEXT,
content_snippet TEXT,
permalink VARCHAR,
file_path VARCHAR,
type VARCHAR,
from_id INTEGER,
to_id INTEGER,
relation_type VARCHAR,
entity_id INTEGER,
category VARCHAR,
metadata JSONB,
created_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE,
textsearchable_index_col tsvector GENERATED ALWAYS AS (
to_tsvector('english', coalesce(title, '') || ' ' || coalesce(content_stems, ''))
) STORED,
PRIMARY KEY (id, type, project_id),
FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE
)
""")
For SQLite: This model is skipped; FTS5 virtual table is created via DDL instead.
For Postgres: This is the actual table structure with tsvector support.
"""
__tablename__ = "search_index"
# Primary key (rowid in SQLite FTS5, explicit id in Postgres)
id = Column(Integer, primary_key=True, autoincrement=True)
# Core searchable fields
title = Column(Text, nullable=True)
content_stems = Column(Text, nullable=True)
content_snippet = Column(Text, nullable=True)
permalink = Column(String(255), nullable=True, index=True)
file_path = Column(Text, nullable=True)
type = Column(String(50), nullable=True)
# Project context
project_id = Column(Integer, nullable=True, index=True)
# Relation fields
from_id = Column(Integer, nullable=True)
to_id = Column(Integer, nullable=True)
relation_type = Column(String(100), nullable=True)
# Observation fields
# Note: FK with CASCADE only applies to Postgres. SQLite uses FTS5 virtual tables
# which don't support foreign keys, so cascade delete is handled explicitly there.
entity_id = Column(Integer, ForeignKey("entity.id", ondelete="CASCADE"), nullable=True)
category = Column(String(100), nullable=True)
# Common fields
# Use JSONB for Postgres, JSON for SQLite
# Note: 'metadata' is a reserved name in SQLAlchemy, so we use 'metadata_' and map to 'metadata'
metadata_ = Column("metadata", JSON().with_variant(JSONB(), "postgresql"), nullable=True)
created_at = Column(DateTime(timezone=True), nullable=True)
updated_at = Column(DateTime(timezone=True), nullable=True)
# Note: textsearchable_index_col (tsvector) will be added by migration for Postgres only
CREATE_POSTGRES_SEARCH_INDEX_FTS = DDL("""
CREATE INDEX IF NOT EXISTS idx_search_index_fts ON search_index USING gin(textsearchable_index_col)
""")
CREATE_POSTGRES_SEARCH_INDEX_METADATA = DDL("""
CREATE INDEX IF NOT EXISTS idx_search_index_metadata_gin ON search_index USING gin(metadata jsonb_path_ops)
""")
# Define FTS5 virtual table creation for SQLite only
# This DDL is executed separately for SQLite databases
@@ -3,7 +3,7 @@
from pathlib import Path
from typing import List, Optional, Sequence, Union, Any
import logfire
from loguru import logger
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
@@ -33,7 +33,6 @@ class EntityRepository(Repository[Entity]):
"""
super().__init__(session_maker, Entity, project_id=project_id)
@logfire.instrument()
async def get_by_id(self, entity_id: int) -> Optional[Entity]:
"""Get entity by numeric ID.
@@ -46,7 +45,6 @@ class EntityRepository(Repository[Entity]):
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, entity_id)
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
"""Get entity by permalink.
@@ -56,7 +54,6 @@ class EntityRepository(Repository[Entity]):
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
return await self.find_one(query)
@logfire.instrument()
async def get_by_title(self, title: str) -> Sequence[Entity]:
"""Get entity by title.
@@ -67,7 +64,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
"""Get entity by file_path.
@@ -85,7 +81,6 @@ class EntityRepository(Repository[Entity]):
# Lightweight methods for permalink resolution (no eager loading)
# -------------------------------------------------------------------------
@logfire.instrument()
async def permalink_exists(self, permalink: str) -> bool:
"""Check if a permalink exists without loading the full entity.
@@ -103,7 +98,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return result.scalar_one_or_none() is not None
@logfire.instrument()
async def get_file_path_for_permalink(self, permalink: str) -> Optional[str]:
"""Get the file_path for a permalink without loading the full entity.
@@ -120,7 +114,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return result.scalar_one_or_none()
@logfire.instrument()
async def get_permalink_for_file_path(self, file_path: Union[Path, str]) -> Optional[str]:
"""Get the permalink for a file_path without loading the full entity.
@@ -137,7 +130,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return result.scalar_one_or_none()
@logfire.instrument()
async def get_all_permalinks(self) -> List[str]:
"""Get all permalinks for this project.
@@ -152,7 +144,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def get_permalink_to_file_path_map(self) -> dict[str, str]:
"""Get a mapping of permalink -> file_path for all entities.
@@ -166,7 +157,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return {row.permalink: row.file_path for row in result.all()}
@logfire.instrument()
async def get_file_path_to_permalink_map(self) -> dict[str, str]:
"""Get a mapping of file_path -> permalink for all entities.
@@ -180,7 +170,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return {row.file_path: row.permalink for row in result.all()}
@logfire.instrument()
async def get_by_file_paths(
self, session: AsyncSession, file_paths: Sequence[Union[Path, str]]
) -> List[Row[Any]]:
@@ -209,7 +198,6 @@ class EntityRepository(Repository[Entity]):
result = await session.execute(query)
return list(result.all())
@logfire.instrument()
async def find_by_checksum(self, checksum: str) -> Sequence[Entity]:
"""Find entities with the given checksum.
@@ -227,7 +215,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def find_by_checksums(self, checksums: Sequence[str]) -> Sequence[Entity]:
"""Find entities with any of the given checksums (batch query for move detection).
@@ -256,7 +243,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool:
"""Delete entity with the provided file_path.
@@ -277,7 +263,6 @@ class EntityRepository(Repository[Entity]):
selectinload(Entity.incoming_relations).selectinload(Relation.to_entity),
]
@logfire.instrument()
async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
"""Find multiple entities by their permalink.
@@ -296,7 +281,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def upsert_entity(self, entity: Entity) -> Entity:
"""Insert or update entity using simple try/catch with database-level conflict resolution.
@@ -398,7 +382,6 @@ class EntityRepository(Repository[Entity]):
entity = await self._handle_permalink_conflict(entity, session)
return entity
@logfire.instrument()
async def get_all_file_paths(self) -> List[str]:
"""Get all file paths for this project - optimized for deletion detection.
@@ -414,7 +397,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def get_distinct_directories(self) -> List[str]:
"""Extract unique directory paths from file_path column.
@@ -443,7 +425,6 @@ class EntityRepository(Repository[Entity]):
return sorted(directories)
@logfire.instrument()
async def find_by_directory_prefix(self, directory_prefix: str) -> Sequence[Entity]:
"""Find entities whose file_path starts with the given directory prefix.
@@ -476,7 +457,6 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
@logfire.instrument()
async def _handle_permalink_conflict(self, entity: Entity, session: AsyncSession) -> Entity:
"""Handle permalink conflicts by generating a unique permalink."""
base_permalink = entity.permalink
@@ -2,7 +2,7 @@
from typing import Dict, List, Sequence
import logfire
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
@@ -22,35 +22,30 @@ class ObservationRepository(Repository[Observation]):
"""
super().__init__(session_maker, Observation, project_id=project_id)
@logfire.instrument()
async def find_by_entity(self, entity_id: int) -> Sequence[Observation]:
"""Find all observations for a specific entity."""
query = select(Observation).filter(Observation.entity_id == entity_id)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_context(self, context: str) -> Sequence[Observation]:
"""Find observations with a specific context."""
query = select(Observation).filter(Observation.context == context)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_category(self, category: str) -> Sequence[Observation]:
"""Find observations with a specific context."""
query = select(Observation).filter(Observation.category == category)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def observation_categories(self) -> Sequence[str]:
"""Return a list of all observation categories."""
query = select(Observation.category).distinct()
result = await self.execute_query(query, use_query_options=False)
return result.scalars().all()
@logfire.instrument()
async def find_by_entities(self, entity_ids: List[int]) -> Dict[int, List[Observation]]:
"""Find all observations for multiple entities in a single query.
@@ -5,7 +5,7 @@ import re
from datetime import datetime
from typing import List, Optional
import logfire
from loguru import logger
from sqlalchemy import text
@@ -26,7 +26,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
- JSONB containment operators for metadata search
"""
@logfire.instrument()
async def init_search_index(self):
"""Create Postgres table with tsvector column and GIN indexes.
@@ -147,7 +146,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
else:
return cleaned_term
@logfire.instrument()
async def search(
self,
search_text: Optional[str] = None,
@@ -260,7 +258,7 @@ class PostgresSearchRepository(SearchRepositoryBase):
{score_expr} as score
FROM search_index
WHERE {where_clause}
ORDER BY score DESC {order_by_clause}
ORDER BY score DESC, id ASC {order_by_clause}
LIMIT :limit
OFFSET :offset
"""
@@ -315,7 +313,6 @@ class PostgresSearchRepository(SearchRepositoryBase):
return results
@logfire.instrument()
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
"""Index multiple items in a single batch operation using UPSERT.
@@ -3,7 +3,7 @@
from pathlib import Path
from typing import Optional, Sequence, Union
import logfire
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -23,7 +23,6 @@ class ProjectRepository(Repository[Project]):
"""Initialize with session maker."""
super().__init__(session_maker, Project)
@logfire.instrument()
async def get_by_name(self, name: str) -> Optional[Project]:
"""Get project by name.
@@ -33,7 +32,6 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.name == name)
return await self.find_one(query)
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> Optional[Project]:
"""Get project by permalink.
@@ -43,7 +41,6 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.permalink == permalink)
return await self.find_one(query)
@logfire.instrument()
async def get_by_path(self, path: Union[Path, str]) -> Optional[Project]:
"""Get project by filesystem path.
@@ -53,7 +50,6 @@ class ProjectRepository(Repository[Project]):
query = self.select().where(Project.path == Path(path).as_posix())
return await self.find_one(query)
@logfire.instrument()
async def get_by_id(self, project_id: int) -> Optional[Project]:
"""Get project by numeric ID.
@@ -66,20 +62,17 @@ class ProjectRepository(Repository[Project]):
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, project_id)
@logfire.instrument()
async def get_default_project(self) -> Optional[Project]:
"""Get the default project (the one marked as is_default=True)."""
query = self.select().where(Project.is_default.is_not(None))
return await self.find_one(query)
@logfire.instrument()
async def get_active_projects(self) -> Sequence[Project]:
"""Get all active projects."""
query = self.select().where(Project.is_active == True) # noqa: E712
result = await self.execute_query(query)
return list(result.scalars().all())
@logfire.instrument()
async def set_as_default(self, project_id: int) -> Optional[Project]:
"""Set a project as the default and unset previous default.
@@ -104,7 +97,6 @@ class ProjectRepository(Repository[Project]):
return target_project
return None # pragma: no cover
@logfire.instrument()
async def update_path(self, project_id: int, new_path: str) -> Optional[Project]:
"""Update project path.
@@ -2,7 +2,7 @@
from typing import Sequence, List, Optional
import logfire
from sqlalchemy import and_, delete, select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
@@ -27,7 +27,6 @@ class RelationRepository(Repository[Relation]):
"""
super().__init__(session_maker, Relation, project_id=project_id)
@logfire.instrument()
async def find_relation(
self, from_permalink: str, to_permalink: str, relation_type: str
) -> Optional[Relation]:
@@ -49,21 +48,18 @@ class RelationRepository(Repository[Relation]):
)
return await self.find_one(query)
@logfire.instrument()
async def find_by_entities(self, from_id: int, to_id: int) -> Sequence[Relation]:
"""Find all relations between two entities."""
query = select(Relation).where((Relation.from_id == from_id) & (Relation.to_id == to_id))
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_by_type(self, relation_type: str) -> Sequence[Relation]:
"""Find all relations of a specific type."""
query = select(Relation).filter(Relation.relation_type == relation_type)
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def delete_outgoing_relations_from_entity(self, entity_id: int) -> None:
"""Delete outgoing relations for an entity.
@@ -73,14 +69,12 @@ class RelationRepository(Repository[Relation]):
async with db.scoped_session(self.session_maker) as session:
await session.execute(delete(Relation).where(Relation.from_id == entity_id))
@logfire.instrument()
async def find_unresolved_relations(self) -> Sequence[Relation]:
"""Find all unresolved relations, where to_id is null."""
query = select(Relation).filter(Relation.to_id.is_(None))
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def find_unresolved_relations_for_entity(self, entity_id: int) -> Sequence[Relation]:
"""Find unresolved relations for a specific entity.
@@ -94,7 +88,6 @@ class RelationRepository(Repository[Relation]):
result = await self.execute_query(query)
return result.scalars().all()
@logfire.instrument()
async def add_all_ignore_duplicates(self, relations: List[Relation]) -> int:
"""Bulk insert relations, ignoring duplicates.
@@ -132,15 +125,22 @@ class RelationRepository(Repository[Relation]):
dialect_name = session.bind.dialect.name if session.bind else "sqlite"
if dialect_name == "postgresql":
stmt = pg_insert(Relation).values(values)
stmt = stmt.on_conflict_do_nothing()
# PostgreSQL: use RETURNING to count inserted rows
# (rowcount is 0 for ON CONFLICT DO NOTHING)
stmt = (
pg_insert(Relation)
.values(values)
.on_conflict_do_nothing()
.returning(Relation.id)
)
result = await session.execute(stmt)
return len(result.fetchall())
else:
# SQLite
# SQLite: rowcount works correctly
stmt = sqlite_insert(Relation).values(values)
stmt = stmt.on_conflict_do_nothing()
result = await session.execute(stmt)
return result.rowcount if result.rowcount else 0
result = await session.execute(stmt)
return result.rowcount if result.rowcount > 0 else 0
def get_load_options(self) -> List[LoaderOption]:
return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]
+1 -16
View File
@@ -2,7 +2,7 @@
from typing import Type, Optional, Any, Sequence, TypeVar, List, Dict
import logfire
from loguru import logger
from sqlalchemy import (
select,
@@ -85,7 +85,6 @@ class Repository[T: Base]:
result = await session.execute(query)
return result.scalars().one_or_none()
@logfire.instrument()
async def select_by_ids(self, session: AsyncSession, ids: List[int]) -> Sequence[T]:
"""Select multiple entities by IDs using an existing session."""
query = (
@@ -97,7 +96,6 @@ class Repository[T: Base]:
result = await session.execute(query)
return result.scalars().all()
@logfire.instrument()
async def add(self, model: T) -> T:
"""
Add a model to the repository. This will also add related objects
@@ -124,7 +122,6 @@ class Repository[T: Base]:
)
return found
@logfire.instrument()
async def add_all(self, models: List[T]) -> Sequence[T]:
"""
Add a list of models to the repository. This will also add related objects
@@ -156,7 +153,6 @@ class Repository[T: Base]:
# Add project filter if applicable
return self._add_project_filter(query)
@logfire.instrument()
async def find_all(
self, skip: int = 0, limit: Optional[int] = None, use_load_options: bool = True
) -> Sequence[T]:
@@ -188,7 +184,6 @@ class Repository[T: Base]:
logger.debug(f"Found {len(items)} {self.Model.__name__} records")
return items
@logfire.instrument()
async def find_by_id(self, entity_id: int) -> Optional[T]:
"""Fetch an entity by its unique identifier."""
logger.debug(f"Finding {self.Model.__name__} by ID: {entity_id}")
@@ -196,7 +191,6 @@ class Repository[T: Base]:
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_id(session, entity_id)
@logfire.instrument()
async def find_by_ids(self, ids: List[int]) -> Sequence[T]:
"""Fetch multiple entities by their identifiers in a single query."""
logger.debug(f"Finding {self.Model.__name__} by IDs: {ids}")
@@ -204,7 +198,6 @@ class Repository[T: Base]:
async with db.scoped_session(self.session_maker) as session:
return await self.select_by_ids(session, ids)
@logfire.instrument()
async def find_one(self, query: Select[tuple[T]]) -> Optional[T]:
"""Execute a query and retrieve a single record."""
# add in load options
@@ -218,7 +211,6 @@ class Repository[T: Base]:
logger.trace(f"No {self.Model.__name__} found")
return entity
@logfire.instrument()
async def create(self, data: dict) -> T:
"""Create a new record from a model instance."""
logger.debug(f"Creating {self.Model.__name__} from entity_data: {data}")
@@ -250,7 +242,6 @@ class Repository[T: Base]:
)
return return_instance
@logfire.instrument()
async def create_all(self, data_list: List[dict]) -> Sequence[T]:
"""Create multiple records in a single transaction."""
logger.debug(f"Bulk creating {len(data_list)} {self.Model.__name__} instances")
@@ -276,7 +267,6 @@ class Repository[T: Base]:
return await self.select_by_ids(session, [model.id for model in model_list]) # pyright: ignore [reportAttributeAccessIssue]
@logfire.instrument()
async def update(self, entity_id: int, entity_data: dict | T) -> Optional[T]:
"""Update an entity with the given data."""
logger.debug(f"Updating {self.Model.__name__} {entity_id} with data: {entity_data}")
@@ -306,7 +296,6 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found to update: {entity_id}")
return None
@logfire.instrument()
async def delete(self, entity_id: int) -> bool:
"""Delete an entity from the database."""
logger.debug(f"Deleting {self.Model.__name__}: {entity_id}")
@@ -324,7 +313,6 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found to delete: {entity_id}")
return False
@logfire.instrument()
async def delete_by_ids(self, ids: List[int]) -> int:
"""Delete records matching given IDs."""
logger.debug(f"Deleting {self.Model.__name__} by ids: {ids}")
@@ -340,7 +328,6 @@ class Repository[T: Base]:
logger.debug(f"Deleted {result.rowcount} records")
return result.rowcount
@logfire.instrument()
async def delete_by_fields(self, **filters: Any) -> bool:
"""Delete records matching given field values."""
logger.debug(f"Deleting {self.Model.__name__} by fields: {filters}")
@@ -357,7 +344,6 @@ class Repository[T: Base]:
logger.debug(f"Deleted {result.rowcount} records")
return deleted
@logfire.instrument()
async def count(self, query: Executable | None = None) -> int:
"""Count entities in the database table."""
async with db.scoped_session(self.session_maker) as session:
@@ -379,7 +365,6 @@ class Repository[T: Base]:
logger.debug(f"Counted {count} {self.Model.__name__} records")
return count
@logfire.instrument()
async def execute_query(
self,
query: Executable,
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any, Dict, List, Optional
import logfire
from loguru import logger
from sqlalchemy import Executable, Result, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -103,7 +103,6 @@ class SearchRepositoryBase(ABC):
"""
pass
@logfire.instrument()
async def index_item(self, search_index_row: SearchIndexRow) -> None:
"""Index or update a single item.
@@ -147,7 +146,6 @@ class SearchRepositoryBase(ABC):
logger.debug(f"indexed row {search_index_row}")
await session.commit()
@logfire.instrument()
async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]) -> None:
"""Index multiple items in a single batch operation.
@@ -195,7 +193,6 @@ class SearchRepositoryBase(ABC):
logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
await session.commit()
@logfire.instrument()
async def delete_by_entity_id(self, entity_id: int) -> None:
"""Delete all search index entries for an entity.
@@ -210,7 +207,6 @@ class SearchRepositoryBase(ABC):
)
await session.commit()
@logfire.instrument()
async def delete_by_permalink(self, permalink: str) -> None:
"""Delete a search index entry by permalink.
@@ -225,7 +221,6 @@ class SearchRepositoryBase(ABC):
)
await session.commit()
@logfire.instrument()
async def execute_query(
self,
query: Executable,
@@ -5,7 +5,7 @@ import re
from datetime import datetime
from typing import List, Optional
import logfire
from loguru import logger
from sqlalchemy import text
@@ -26,7 +26,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
- Prefix wildcard matching with *
"""
@logfire.instrument()
async def init_search_index(self):
"""Create FTS5 virtual table for search.
@@ -281,7 +280,6 @@ class SQLiteSearchRepository(SearchRepositoryBase):
# For non-Boolean queries, use the single term preparation logic
return self._prepare_single_term(term, is_prefix)
@logfire.instrument()
async def search(
self,
search_text: Optional[str] = None,
+1 -3
View File
@@ -4,7 +4,7 @@ from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import List, Optional, Tuple
import logfire
from loguru import logger
from sqlalchemy import text
@@ -86,7 +86,6 @@ class ContextService:
self.entity_repository = entity_repository
self.observation_repository = observation_repository
@logfire.instrument()
async def build_context(
self,
memory_url: Optional[MemoryUrl] = None,
@@ -217,7 +216,6 @@ class ContextService:
# Return the structured ContextResult
return ContextResult(results=context_results, metadata=metadata)
@logfire.instrument()
async def find_related(
self,
type_id_pairs: List[Tuple[str, int]],
@@ -6,7 +6,6 @@ import os
from datetime import datetime
from typing import Dict, List, Optional, Sequence
import logfire
from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
@@ -37,7 +36,6 @@ class DirectoryService:
"""
self.entity_repository = entity_repository
@logfire.instrument()
async def get_directory_tree(self) -> DirectoryNode:
"""Build a hierarchical directory tree from indexed files."""
@@ -105,7 +103,6 @@ class DirectoryService:
# Return the root node with its children
return root_node
@logfire.instrument()
async def get_directory_structure(self) -> DirectoryNode:
"""Build a hierarchical directory structure without file details.
@@ -149,7 +146,6 @@ class DirectoryService:
return root_node
@logfire.instrument()
async def list_directory(
self,
dir_name: str = "/",
+1 -18
View File
@@ -7,7 +7,7 @@ import frontmatter
import yaml
from loguru import logger
from sqlalchemy.exc import IntegrityError
import logfire
from basic_memory.config import ProjectConfig, BasicMemoryConfig
from basic_memory.file_utils import (
@@ -53,7 +53,6 @@ class EntityService(BaseService[EntityModel]):
self.link_resolver = link_resolver
self.app_config = app_config
@logfire.instrument()
async def detect_file_path_conflicts(
self, file_path: str, skip_check: bool = False
) -> List[Entity]:
@@ -93,7 +92,6 @@ class EntityService(BaseService[EntityModel]):
return conflicts
@logfire.instrument()
async def resolve_permalink(
self,
file_path: Permalink | Path,
@@ -160,7 +158,6 @@ class EntityService(BaseService[EntityModel]):
return permalink
@logfire.instrument()
async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
"""Create new entity or update existing one.
Returns: (entity, is_new) where is_new is True if a new entity was created
@@ -182,7 +179,6 @@ class EntityService(BaseService[EntityModel]):
# Create new entity
return await self.create_entity(schema), True
@logfire.instrument()
async def create_entity(self, schema: EntitySchema) -> EntityModel:
"""Create a new entity and write to filesystem."""
logger.debug(f"Creating entity: {schema.title}")
@@ -249,7 +245,6 @@ class EntityService(BaseService[EntityModel]):
# Set final checksum to mark complete
return await self.repository.update(entity.id, {"checksum": checksum})
@logfire.instrument()
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
"""Update an entity's content and metadata."""
logger.debug(
@@ -330,7 +325,6 @@ class EntityService(BaseService[EntityModel]):
return entity
@logfire.instrument()
async def delete_entity(self, permalink_or_id: str | int) -> bool:
"""Delete entity and its file."""
logger.debug(f"Deleting entity: {permalink_or_id}")
@@ -360,7 +354,6 @@ class EntityService(BaseService[EntityModel]):
logger.info(f"Entity not found: {permalink_or_id}")
return True # Already deleted
@logfire.instrument()
async def get_by_permalink(self, permalink: str) -> EntityModel:
"""Get entity by type and name combination."""
logger.debug(f"Getting entity by permalink: {permalink}")
@@ -369,24 +362,20 @@ class EntityService(BaseService[EntityModel]):
raise EntityNotFoundError(f"Entity not found: {permalink}")
return db_entity
@logfire.instrument()
async def get_entities_by_id(self, ids: List[int]) -> Sequence[EntityModel]:
"""Get specific entities and their relationships."""
logger.debug(f"Getting entities: {ids}")
return await self.repository.find_by_ids(ids)
@logfire.instrument()
async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
"""Get specific nodes and their relationships."""
logger.debug(f"Getting entities permalinks: {permalinks}")
return await self.repository.find_by_permalinks(permalinks)
@logfire.instrument()
async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None:
"""Delete entity by file path."""
await self.repository.delete_by_file_path(str(file_path))
@logfire.instrument()
async def create_entity_from_markdown(
self, file_path: Path, markdown: EntityMarkdown
) -> EntityModel:
@@ -412,7 +401,6 @@ class EntityService(BaseService[EntityModel]):
logger.error(f"Failed to upsert entity for {file_path}: {e}")
raise EntityCreationError(f"Failed to create entity: {str(e)}") from e
@logfire.instrument()
async def update_entity_and_observations(
self, file_path: Path, markdown: EntityMarkdown
) -> EntityModel:
@@ -454,7 +442,6 @@ class EntityService(BaseService[EntityModel]):
db_entity,
)
@logfire.instrument()
async def update_entity_relations(
self,
path: str,
@@ -527,7 +514,6 @@ class EntityService(BaseService[EntityModel]):
return await self.repository.get_by_file_path(path)
@logfire.instrument()
async def edit_entity(
self,
identifier: str,
@@ -585,7 +571,6 @@ class EntityService(BaseService[EntityModel]):
return entity
@logfire.instrument()
def apply_edit_operation(
self,
current_content: str,
@@ -638,7 +623,6 @@ class EntityService(BaseService[EntityModel]):
else:
raise ValueError(f"Unsupported operation: {operation}")
@logfire.instrument()
def replace_section_content(
self, current_content: str, section_header: str, new_content: str
) -> str:
@@ -758,7 +742,6 @@ class EntityService(BaseService[EntityModel]):
return content + "\n" + current_content # pragma: no cover
return content + current_content # pragma: no cover
@logfire.instrument()
async def move_entity(
self,
identifier: str,
+1 -11
View File
@@ -8,7 +8,7 @@ from pathlib import Path
from typing import Any, Dict, Tuple, Union
import aiofiles
import logfire
import yaml
from basic_memory import file_utils
@@ -60,7 +60,6 @@ class FileService:
"""
return self.base_path / entity.file_path
@logfire.instrument()
async def read_entity_content(self, entity: EntityModel) -> str:
"""Get entity's content without frontmatter or structured sections.
@@ -79,7 +78,6 @@ class FileService:
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
@logfire.instrument()
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem.
@@ -92,7 +90,6 @@ class FileService:
path = self.get_entity_path(entity)
await self.delete_file(path)
@logfire.instrument()
async def exists(self, path: FilePath) -> bool:
"""Check if file exists at the provided path.
@@ -119,7 +116,6 @@ class FileService:
logger.error("Failed to check file existence", path=str(path), error=str(e))
raise FileOperationError(f"Failed to check file existence: {e}")
@logfire.instrument()
async def ensure_directory(self, path: FilePath) -> None:
"""Ensure directory exists, creating if necessary.
@@ -147,7 +143,6 @@ class FileService:
logger.error("Failed to create directory", path=str(path), error=str(e))
raise FileOperationError(f"Failed to create directory {path}: {e}")
@logfire.instrument()
async def write_file(self, path: FilePath, content: str) -> str:
"""Write content to file and return checksum.
@@ -191,7 +186,6 @@ class FileService:
logger.exception("File write error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to write file: {e}")
@logfire.instrument()
async def read_file_content(self, path: FilePath) -> str:
"""Read file content using true async I/O with aiofiles.
@@ -227,7 +221,6 @@ class FileService:
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
@logfire.instrument()
async def read_file(self, path: FilePath) -> Tuple[str, str]:
"""Read file and compute checksum using true async I/O.
@@ -270,7 +263,6 @@ class FileService:
logger.exception("File read error", path=str(full_path), error=str(e))
raise FileOperationError(f"Failed to read file: {e}")
@logfire.instrument()
async def delete_file(self, path: FilePath) -> None:
"""Delete file if it exists.
@@ -285,7 +277,6 @@ class FileService:
full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
full_path.unlink(missing_ok=True)
@logfire.instrument()
async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
"""Update frontmatter fields in a file while preserving all content.
@@ -354,7 +345,6 @@ class FileService:
)
raise FileOperationError(f"Failed to update frontmatter: {e}")
@logfire.instrument()
async def compute_checksum(self, path: FilePath) -> str:
"""Compute checksum for a file using true async I/O.
+1 -5
View File
@@ -8,7 +8,7 @@ import asyncio
import os
from pathlib import Path
import logfire
from loguru import logger
from basic_memory import db
@@ -19,7 +19,6 @@ from basic_memory.repository import (
)
@logfire.instrument()
async def initialize_database(app_config: BasicMemoryConfig) -> None:
"""Initialize database with migrations handled automatically by get_or_create_db.
@@ -41,7 +40,6 @@ async def initialize_database(app_config: BasicMemoryConfig) -> None:
# more specific error if the database is actually unusable
@logfire.instrument()
async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
"""Ensure all projects in config.json exist in the projects table and vice versa.
@@ -75,7 +73,6 @@ async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
logger.info("Continuing with initialization despite synchronization error")
@logfire.instrument()
async def initialize_file_sync(
app_config: BasicMemoryConfig,
):
@@ -152,7 +149,6 @@ async def initialize_file_sync(
return None
@logfire.instrument()
async def initialize_app(
app_config: BasicMemoryConfig,
):
+1 -2
View File
@@ -2,7 +2,7 @@
from typing import Optional, Tuple
import logfire
from loguru import logger
from basic_memory.models import Entity
@@ -27,7 +27,6 @@ class LinkResolver:
self.entity_repository = entity_repository
self.search_service = search_service
@logfire.instrument()
async def resolve_link(
self, link_text: str, use_search: bool = True, strict: bool = False
) -> Optional[Entity]:
+1 -13
View File
@@ -8,7 +8,7 @@ from datetime import datetime
from pathlib import Path
from typing import Dict, Optional, Sequence
import logfire
from loguru import logger
from sqlalchemy import text
@@ -82,7 +82,6 @@ class ProjectService:
"""
return os.environ.get("BASIC_MEMORY_PROJECT", self.config_manager.default_project)
@logfire.instrument()
async def list_projects(self) -> Sequence[Project]:
"""List all projects without loading entity relationships.
@@ -92,7 +91,6 @@ class ProjectService:
"""
return await self.repository.find_all(use_load_options=False)
@logfire.instrument()
async def get_project(self, name: str) -> Optional[Project]:
"""Get the file path for a project by name or permalink."""
return await self.repository.get_by_name(name) or await self.repository.get_by_permalink(
@@ -133,7 +131,6 @@ class ProjectService:
# Not nested in either direction
return False
@logfire.instrument()
async def add_project(self, name: str, path: str, set_default: bool = False) -> None:
"""Add a new project to the configuration and database.
@@ -225,7 +222,6 @@ class ProjectService:
logger.info(f"Project '{name}' added at {resolved_path}")
@logfire.instrument()
async def remove_project(self, name: str, delete_notes: bool = False) -> None:
"""Remove a project from configuration and database.
@@ -276,7 +272,6 @@ class ProjectService:
except Exception as e:
logger.warning(f"Failed to delete project directory {project_path}: {e}")
@logfire.instrument()
async def set_default_project(self, name: str) -> None:
"""Set the default project in configuration and database.
@@ -301,7 +296,6 @@ class ProjectService:
logger.info(f"Project '{name}' set as default in configuration and database")
@logfire.instrument()
async def _ensure_single_default_project(self) -> None:
"""Ensure only one project has is_default=True.
@@ -343,7 +337,6 @@ class ProjectService:
f"Set '{config_default}' as default project (was missing)"
) # pragma: no cover
@logfire.instrument()
async def synchronize_projects(self) -> None: # pragma: no cover
"""Synchronize projects between database and configuration.
@@ -428,7 +421,6 @@ class ProjectService:
logger.info("Project synchronization complete")
@logfire.instrument()
async def move_project(self, name: str, new_path: str) -> None:
"""Move a project to a new location.
@@ -470,7 +462,6 @@ class ProjectService:
self.config_manager.save_config(config)
raise ValueError(f"Project '{name}' not found in database")
@logfire.instrument()
async def update_project( # pragma: no cover
self, name: str, updated_path: Optional[str] = None, is_active: Optional[bool] = None
) -> None:
@@ -530,7 +521,6 @@ class ProjectService:
f"Changed default project to '{new_default.name}' as '{name}' was deactivated"
)
@logfire.instrument()
async def get_project_info(self, project_name: Optional[str] = None) -> ProjectInfoResponse:
"""Get comprehensive information about the specified Basic Memory project.
@@ -598,7 +588,6 @@ class ProjectService:
system=system,
)
@logfire.instrument()
async def get_statistics(self, project_id: int) -> ProjectStatistics:
"""Get statistics about the specified project.
@@ -715,7 +704,6 @@ class ProjectService:
isolated_entities=isolated_count,
)
@logfire.instrument()
async def get_activity_metrics(self, project_id: int) -> ActivityMetrics:
"""Get activity metrics for the specified project.
+1 -11
View File
@@ -4,7 +4,7 @@ import ast
from datetime import datetime
from typing import List, Optional, Set
import logfire
from dateparser import parse
from fastapi import BackgroundTasks
from loguru import logger
@@ -51,12 +51,10 @@ class SearchService:
self.entity_repository = entity_repository
self.file_service = file_service
@logfire.instrument()
async def init_search_index(self):
"""Create FTS5 virtual table if it doesn't exist."""
await self.repository.init_search_index()
@logfire.instrument()
async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
"""Reindex all content from database."""
@@ -73,7 +71,6 @@ class SearchService:
logger.info("Reindex complete")
@logfire.instrument()
async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
"""Search across all indexed content.
@@ -171,7 +168,6 @@ class SearchService:
return [] # pragma: no cover
@logfire.instrument()
async def index_entity(
self,
entity: Entity,
@@ -183,7 +179,6 @@ class SearchService:
else:
await self.index_entity_data(entity, content)
@logfire.instrument()
async def index_entity_data(
self,
entity: Entity,
@@ -197,7 +192,6 @@ class SearchService:
entity, content
) if entity.is_markdown else await self.index_entity_file(entity)
@logfire.instrument()
async def index_entity_file(
self,
entity: Entity,
@@ -220,7 +214,6 @@ class SearchService:
)
)
@logfire.instrument()
async def index_entity_markdown(
self,
entity: Entity,
@@ -373,17 +366,14 @@ class SearchService:
# Batch insert all rows at once
await self.repository.bulk_index_items(rows_to_index)
@logfire.instrument()
async def delete_by_permalink(self, permalink: str):
"""Delete an item from the search index."""
await self.repository.delete_by_permalink(permalink)
@logfire.instrument()
async def delete_by_entity_id(self, entity_id: int):
"""Delete an item from the search index."""
await self.repository.delete_by_entity_id(entity_id)
@logfire.instrument()
async def handle_delete(self, entity: Entity):
"""Handle complete entity deletion from search index including observations and relations.
+41 -79
View File
@@ -10,7 +10,7 @@ from pathlib import Path
from typing import AsyncIterator, Dict, List, Optional, Set, Tuple
import aiofiles.os
import logfire
from loguru import logger
from sqlalchemy.exc import IntegrityError
@@ -215,17 +215,12 @@ class SyncService:
f"path={path}, error={error}"
)
# Record metric for file failure
logfire.metric_counter("sync.circuit_breaker.failures").add(1)
# Log when threshold is reached
if failure_info.count >= MAX_CONSECUTIVE_FAILURES:
logger.error(
f"File {path} has failed {MAX_CONSECUTIVE_FAILURES} times and will be skipped. "
f"First failure: {failure_info.first_failure}, Last error: {error}"
)
# Record metric for file being blocked by circuit breaker
logfire.metric_counter("sync.circuit_breaker.blocked_files").add(1)
else:
# Create new failure record
self._file_failures[path] = FileFailureInfo(
@@ -255,7 +250,6 @@ class SyncService:
logger.info(f"Clearing failure history for {path} after successful sync")
del self._file_failures[path]
@logfire.instrument()
async def sync(
self, directory: Path, project_name: Optional[str] = None, force_full: bool = False
) -> SyncReport:
@@ -282,63 +276,58 @@ class SyncService:
)
# sync moves first
with logfire.span("process_moves", move_count=len(report.moves)):
for old_path, new_path in report.moves.items():
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified:
report.modified.remove(new_path)
logger.debug(
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
)
else:
await self.handle_move(old_path, new_path)
for old_path, new_path in report.moves.items():
# in the case where a file has been deleted and replaced by another file
# it will show up in the move and modified lists, so handle it in modified
if new_path in report.modified:
report.modified.remove(new_path)
logger.debug(
f"File marked as moved and modified: old_path={old_path}, new_path={new_path}"
)
else:
await self.handle_move(old_path, new_path)
# deleted next
with logfire.span("process_deletes", delete_count=len(report.deleted)):
for path in report.deleted:
await self.handle_delete(path)
for path in report.deleted:
await self.handle_delete(path)
# then new and modified
with logfire.span("process_new_files", new_count=len(report.new)):
for path in report.new:
entity, _ = await self.sync_file(path, new=True)
for path in report.new:
entity, _ = await self.sync_file(path, new=True)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
with logfire.span("process_modified_files", modified_count=len(report.modified)):
for path in report.modified:
entity, _ = await self.sync_file(path, new=False)
for path in report.modified:
entity, _ = await self.sync_file(path, new=False)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
# Track if file was skipped
if entity is None and await self._should_skip_file(path):
failure_info = self._file_failures[path]
report.skipped_files.append(
SkippedFile(
path=path,
reason=failure_info.last_error,
failure_count=failure_info.count,
first_failed=failure_info.first_failure,
)
)
# Only resolve relations if there were actual changes
# If no files changed, no new unresolved relations could have been created
with logfire.span("resolve_relations"):
if report.total > 0:
await self.resolve_relations()
else:
logger.info("Skipping relation resolution - no file changes detected")
if report.total > 0:
await self.resolve_relations()
else:
logger.info("Skipping relation resolution - no file changes detected")
# Update scan watermark after successful sync
# Use the timestamp from sync start (not end) to ensure we catch files
@@ -361,15 +350,6 @@ class SyncService:
duration_ms = int((time.time() - start_time) * 1000)
# Record metrics for sync operation
logfire.metric_histogram("sync.duration", unit="ms").record(duration_ms)
logfire.metric_counter("sync.files.new").add(len(report.new))
logfire.metric_counter("sync.files.modified").add(len(report.modified))
logfire.metric_counter("sync.files.deleted").add(len(report.deleted))
logfire.metric_counter("sync.files.moved").add(len(report.moves))
if report.skipped_files:
logfire.metric_counter("sync.files.skipped").add(len(report.skipped_files))
# Log summary with skipped files if any
if report.skipped_files:
logger.warning(
@@ -390,7 +370,6 @@ class SyncService:
return report
@logfire.instrument()
async def scan(self, directory, force_full: bool = False):
"""Smart scan using watermark and file count for large project optimization.
@@ -472,12 +451,6 @@ class SyncService:
logger.warning("No scan watermark available, falling back to full scan")
file_paths_to_scan = await self._scan_directory_full(directory)
# Record scan type metric
logfire.metric_counter(f"sync.scan.{scan_type}").add(1)
logfire.metric_histogram("sync.scan.files_scanned", unit="files").record(
len(file_paths_to_scan)
)
# Step 3: Process each file with mtime-based comparison
scanned_paths: Set[str] = set()
changed_checksums: Dict[str, str] = {}
@@ -589,7 +562,6 @@ class SyncService:
report.checksums = changed_checksums
scan_duration_ms = int((time.time() - scan_start_time) * 1000)
logfire.metric_histogram("sync.scan.duration", unit="ms").record(scan_duration_ms)
logger.info(
f"Completed {scan_type} scan for directory {directory} in {scan_duration_ms}ms, "
@@ -599,7 +571,6 @@ class SyncService:
)
return report
@logfire.instrument()
async def sync_file(
self, path: str, new: bool = True
) -> Tuple[Optional[Entity], Optional[str]]:
@@ -654,7 +625,6 @@ class SyncService:
return None, None
@logfire.instrument()
async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a markdown file with full processing.
@@ -737,7 +707,6 @@ class SyncService:
# Return the final checksum to ensure everything is consistent
return entity, final_checksum
@logfire.instrument()
async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
"""Sync a non-markdown file with basic tracking.
@@ -838,7 +807,6 @@ class SyncService:
return updated, checksum
@logfire.instrument()
async def handle_delete(self, file_path: str):
"""Handle complete entity deletion including search index cleanup."""
@@ -870,7 +838,6 @@ class SyncService:
else:
await self.search_service.delete_by_entity_id(entity.id)
@logfire.instrument()
async def handle_move(self, old_path, new_path):
logger.debug("Moving entity", old_path=old_path, new_path=new_path)
@@ -975,7 +942,6 @@ class SyncService:
# update search index
await self.search_service.index_entity(updated)
@logfire.instrument()
async def resolve_relations(self, entity_id: int | None = None):
"""Try to resolve unresolved relations.
@@ -1074,8 +1040,6 @@ class SyncService:
f"error: {error_msg}. Falling back to manual count. "
f"This will slow down watermark detection!"
)
# Track optimization failures for visibility
logfire.metric_counter("sync.scan.file_count_failure").add(1)
# Fallback: count using scan_directory
count = 0
async for _ in self.scan_directory(directory):
@@ -1116,8 +1080,6 @@ class SyncService:
f"error: {error_msg}. Falling back to full scan. "
f"This will cause slow syncs on large projects!"
)
# Track optimization failures for visibility
logfire.metric_counter("sync.scan.optimization_failure").add(1)
# Fallback to full scan
return await self._scan_directory_full(directory)
+20 -5
View File
@@ -5,7 +5,10 @@ import os
from collections import defaultdict
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Set, Sequence
from typing import List, Optional, Set, Sequence, Callable, Awaitable, TYPE_CHECKING
if TYPE_CHECKING:
from basic_memory.sync.sync_service import SyncService
from basic_memory.config import BasicMemoryConfig, WATCH_STATUS_JSON
from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
@@ -71,12 +74,17 @@ class WatchServiceState(BaseModel):
self.last_error = datetime.now()
# Type alias for sync service factory function
SyncServiceFactory = Callable[[Project], Awaitable["SyncService"]]
class WatchService:
def __init__(
self,
app_config: BasicMemoryConfig,
project_repository: ProjectRepository,
quiet: bool = False,
sync_service_factory: Optional[SyncServiceFactory] = None,
):
self.app_config = app_config
self.project_repository = project_repository
@@ -84,10 +92,20 @@ class WatchService:
self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
self.status_path.parent.mkdir(parents=True, exist_ok=True)
self._ignore_patterns_cache: dict[Path, Set[str]] = {}
self._sync_service_factory = sync_service_factory
# quiet mode for mcp so it doesn't mess up stdout
self.console = Console(quiet=quiet)
async def _get_sync_service(self, project: Project) -> "SyncService":
"""Get sync service for a project, using factory if provided."""
if self._sync_service_factory:
return await self._sync_service_factory(project)
# Fall back to default factory
from basic_memory.sync.sync_service import get_sync_service
return await get_sync_service(project)
async def _schedule_restart(self, stop_event: asyncio.Event):
"""Schedule a restart of the watch service after the configured interval."""
await asyncio.sleep(self.app_config.watch_project_reload_interval)
@@ -233,9 +251,6 @@ class WatchService:
async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None:
"""Process a batch of file changes"""
# avoid circular imports
from basic_memory.sync.sync_service import get_sync_service
# Check if project still exists in configuration before processing
# This prevents deleted projects from being recreated by background sync
from basic_memory.config import ConfigManager
@@ -250,7 +265,7 @@ class WatchService:
)
return
sync_service = await get_sync_service(project)
sync_service = await self._get_sync_service(project)
file_service = sync_service.file_service
start_time = time.time()
-3
View File
@@ -67,9 +67,6 @@ class PathLike(Protocol):
# This preserves compatibility with existing code while we migrate
FilePath = Union[Path, str]
# Disable the "Queue is full" warning
logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
def generate_permalink(file_path: Union[Path, str, PathLike], split_extension: bool = True) -> str:
"""Generate a stable permalink from a file path.