Merge pull request #4 from basicmachines-co/db-init

Create/update db on startup

README.md

@@ -113,12 +113,12 @@ Following relation 'relates_to [[Platform Support]]':
 ```
 
 Each related document can lead to more context, building a rich semantic understanding of your knowledge base. All of
-this context comes from standard markdown files that both humans and LLMs can read and write.h
+this context comes from standard markdown files that both humans and LLMs can read and write.
 
 Everything stays in local markdown files that you can:
 
 - Edit in any text editor
-- Put in git
+- Version via git
 - Back up normally
 - Share when you want to
 

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "dateparser>=1.2.0",
     "watchfiles>=1.0.4",
     "fastapi[standard]>=0.115.8",
+    "alembic>=1.14.1",
 ]
 
 [project.optional-dependencies]

src/basic_memory/api/app.py

@@ -7,18 +7,40 @@ from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
 from basic_memory import db
-from .routers import knowledge, search, memory, resource
+from basic_memory.api.routers import knowledge, search, memory, resource
+from basic_memory.config import config
+from basic_memory.services import DatabaseService
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Lifecycle manager for the FastAPI app."""
     logger.info("Starting Basic Memory API")
+    
+    # check the db state
+    await check_db(app)
     yield
     logger.info("Shutting down Basic Memory API")
     await db.shutdown_db()
 
 
+async def check_db(app: FastAPI):
+    logger.info("Checking database state")
+
+    # Initialize DB management service
+    db_service = DatabaseService(
+        config=config,
+    )
+
+    # Check and initialize DB if needed
+    if not await db_service.check_db():
+        raise RuntimeError("Database initialization failed")
+
+    # Clean up old backups on shutdown
+    await db_service.cleanup_backups()
+
+
+
 # Initialize FastAPI app
 app = FastAPI(
     title="Basic Memory API",

src/basic_memory/config.py

@@ -37,7 +37,11 @@ class ProjectConfig(BaseSettings):
     @property
     def database_path(self) -> Path:
         """Get SQLite database path."""
-        return self.home / DATA_DIR_NAME / DATABASE_NAME
+        database_path = self.home / DATA_DIR_NAME / DATABASE_NAME
+        if not database_path.exists():
+            database_path.parent.mkdir(parents=True, exist_ok=True)
+            database_path.touch()
+        return database_path
 
     @field_validator("home")
     @classmethod

src/basic_memory/db.py

@@ -14,8 +14,7 @@ from sqlalchemy.ext.asyncio import (
     async_scoped_session,
 )
 
-from basic_memory.models import Base
-
+from basic_memory.models import Base, SCHEMA_VERSION
 
 # Module level state
 _engine: Optional[AsyncEngine] = None
@@ -69,12 +68,31 @@ async def scoped_session(
         await factory.remove()
 
 
-async def init_db(session: AsyncSession):
+async def init_db() -> None:
     """Initialize database with required tables."""
-    await session.execute(text("PRAGMA foreign_keys=ON"))
-    conn = await session.connection()
-    await conn.run_sync(Base.metadata.create_all)
-    await session.commit()
+
+    logger.info("Initializing database...")
+
+    async with scoped_session(_session_maker) as session:
+        await session.execute(text("PRAGMA foreign_keys=ON"))
+        conn = await session.connection()
+        await conn.run_sync(Base.metadata.create_all)
+        
+        await session.commit()
+
+async def drop_db():
+    """Drop all database tables."""
+    global _engine, _session_maker
+    
+    logger.info("Dropping tables...")
+    async with scoped_session(_session_maker) as session:
+        conn = await session.connection()
+        await conn.run_sync(Base.metadata.drop_all)
+        await session.commit()
+        
+    # reset global engine and session_maker
+    _engine = None
+    _session_maker = None
 
 
 async def get_or_create_db(
@@ -83,7 +101,7 @@ async def get_or_create_db(
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
     """Get or create database engine and session maker."""
     global _engine, _session_maker
-    
+
     if _engine is None:
         db_url = DatabaseType.get_db_url(db_path, db_type)
         logger.debug(f"Creating engine for db_url: {db_url}")
@@ -91,9 +109,7 @@ async def get_or_create_db(
         _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 
         # Initialize database
-        logger.debug("Initializing database...")
-        async with scoped_session(_session_maker) as db_session:
-            await init_db(db_session)
+        await init_db()
 
     return _engine, _session_maker
 
@@ -101,35 +117,38 @@ async def get_or_create_db(
 async def shutdown_db():
     """Clean up database connections."""
     global _engine, _session_maker
-    
+
     if _engine:
         await _engine.dispose()
         _engine = None
         _session_maker = None
 
 
+
 @asynccontextmanager
 async def engine_session_factory(
     db_path: Path,
-    db_type: DatabaseType = DatabaseType.FILESYSTEM,
+    db_type: DatabaseType = DatabaseType.MEMORY,
     init: bool = True,
 ) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
     """Create engine and session factory.
-    
+
     Note: This is primarily used for testing where we want a fresh database
     for each test. For production use, use get_or_create_db() instead.
     """
+
+    global _engine, _session_maker
+    
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
-    engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    
+    _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
     try:
-        factory = async_sessionmaker(engine, expire_on_commit=False)
+        _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 
         if init:
-            logger.debug("Initializing database...")
-            async with scoped_session(factory) as db_session:
-                await init_db(db_session)
+            await init_db()
 
-        yield engine, factory
+        yield _engine, _session_maker
     finally:
-        await engine.dispose()
+        await _engine.dispose()

src/basic_memory/file_utils.py

@@ -170,79 +170,44 @@ def remove_frontmatter(content: str) -> str:
         raise
 
 
-def remove_frontmatter_lenient(content: str) -> str:
-    """
-    Remove frontmatter markers and anything between them without validation.
-    
-    This is a more permissive version of remove_frontmatter that doesn't
-    try to validate the YAML content. It simply removes everything between
-    the first two '---' markers if they exist.
+async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
+    """Update frontmatter fields in a file while preserving all content.
+
+    Only modifies the frontmatter section, leaving all content untouched.
+    Creates frontmatter section if none exists.
+    Returns checksum of updated file.
 
     Args:
-        content: Content that may contain frontmatter
+        path: Path to markdown file
+        updates: Dict of frontmatter fields to update
 
     Returns:
-        Content with any frontmatter markers and content removed
-    """
-    content = content.strip()
-    if not content.startswith("---"):
-        return content
-
-    # Find the second marker
-    rest = content[3:].strip()
-    if "---" not in rest:
-        return content
-
-    # Split on the second marker and take everything after
-    parts = rest.split("---", 1)
-    return parts[1].strip()
-
-
-async def add_frontmatter(content: str, frontmatter: Dict[str, Any]) -> str:
-    """
-    Add YAML frontmatter to content.
-    
-    Args:
-        content: Main content text
-        frontmatter: Key-value pairs for frontmatter
-        
-    Returns:
-        Content with YAML frontmatter prepended
-        
-    Raises:
-        ParseError: If YAML serialization fails
-    """
-    try:
-        yaml_fm = yaml.dump(frontmatter, sort_keys=False)
-        return f"---\n{yaml_fm}---\n\n{content.strip()}"
-    except yaml.YAMLError as e:
-        logger.error(f"Failed to add frontmatter: {e}")
-        raise ParseError(f"Failed to add frontmatter: {e}")
-
-
-async def parse_content_with_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
-    """
-    Parse both frontmatter and content.
-
-    Args:
-        content: Text content with optional frontmatter
-
-    Returns:
-        Tuple of (frontmatter dict, content without frontmatter)
+        Checksum of updated file
 
     Raises:
-        ParseError: If parsing fails
+        FileError: If file operations fail
+        ParseError: If frontmatter parsing fails
     """
     try:
-        if not has_frontmatter(content):
-            return {}, content.strip()
+        # Read current content
+        content = path.read_text()
 
-        frontmatter = parse_frontmatter(content)
-        remaining = remove_frontmatter(content)
-        return frontmatter, remaining
+        # Parse current frontmatter
+        current_fm = {}
+        if has_frontmatter(content):
+            current_fm = parse_frontmatter(content)
+            content = remove_frontmatter(content)
+
+        # Update frontmatter
+        new_fm = {**current_fm, **updates}
+
+        # Write new file with updated frontmatter
+        yaml_fm = yaml.dump(new_fm, sort_keys=False)
+        final_content = f"---\n{yaml_fm}---\n\n{content.strip()}"
+
+        await write_file_atomic(path, final_content)
+        return await compute_checksum(final_content)
 
     except Exception as e:
-        if not isinstance(e, ParseError):
-            logger.error(f"Failed to parse content with frontmatter: {e}")
-            raise ParseError(f"Failed to parse content with frontmatter: {e}")
-        raise
+        logger.error(f"Failed to update frontmatter in {path}: {e}")
+        raise FileError(f"Failed to update frontmatter: {e}")

src/basic_memory/markdown/entity_parser.py

@@ -116,7 +116,7 @@ class EntityParser:
 
         metadata = post.metadata
         metadata["title"] = post.metadata.get("title", file_path.name)
-        metadata["type"] = metadata.get("type", "note")
+        metadata["type"] = post.metadata.get("type", "note")
         metadata["tags"] = parse_tags(post.metadata.get("tags", []))
 
         # frontmatter

src/basic_memory/markdown/markdown_processor.py

@@ -1,17 +1,3 @@
-"""Process markdown files with structured sections.
-
-This module follows a Read -> Modify -> Write pattern for all file operations:
-1. Read entire file and parse into EntityMarkdown schema
-2. Modify the schema (add relation, update content, etc)
-3. Write entire file atomically using temp file + swap
-
-No in-place updates are performed. Each write reconstructs the entire file from the schema.
-The file format has two distinct types of content:
-1. User content - Free form text that is preserved exactly as written
-2. Structured sections - Observations and Relations that are always formatted
-   in a standard way and can be overwritten since they're tracked in our schema
-"""
-
 from pathlib import Path
 from typing import Optional
 from collections import OrderedDict
@@ -33,6 +19,8 @@ class DirtyFileError(Exception):
 
 class MarkdownProcessor:
     """Process markdown files while preserving content and structure.
+    
+    used only for import
 
     This class handles the file I/O aspects of our markdown processing. It:
     1. Uses EntityParser for reading/parsing files into our schema

src/basic_memory/models/__init__.py

@@ -1,12 +1,15 @@
 """Models package for basic-memory."""
 
+import basic_memory
 from basic_memory.models.base import Base
 from basic_memory.models.knowledge import Entity, Observation, Relation, ObservationCategory
 
+SCHEMA_VERSION = basic_memory.__version__ + "-" + "003"
+
 __all__ = [
-    'Base',
-    'Entity',
-    'Observation', 
-    'ObservationCategory', 
-    'Relation'
-]
+    "Base",
+    "Entity",
+    "Observation",
+    "ObservationCategory",
+    "Relation",
+]

src/basic_memory/models/base.py

@@ -1,9 +1,10 @@
 """Base model class for SQLAlchemy models."""
-
+from sqlalchemy import String, Integer
 from sqlalchemy.ext.asyncio import AsyncAttrs
-from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
 class Base(AsyncAttrs, DeclarativeBase):
     """Base class for all models"""
-    pass
+    pass
+

src/basic_memory/repository/search_repository.py

@@ -173,7 +173,7 @@ class SearchRepository:
             LIMIT :limit
         """
 
-        logger.debug(f"Search {sql} params: {params}")
+        #logger.debug(f"Search {sql} params: {params}")
         async with db.scoped_session(self.session_maker) as session:
             result = await session.execute(text(sql), params)
             rows = result.fetchall()
@@ -199,8 +199,9 @@ class SearchRepository:
             for row in rows
         ]
 
-        for r in results:
-            logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
+        #for r in results:
+        #    logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
+        
         return results
 
     async def index_item(

src/basic_memory/services/__init__.py

@@ -1,5 +1,5 @@
 """Services package."""
-
+from .database_service import DatabaseService
 from .service import BaseService
 from .file_service import FileService
 from .entity_service import EntityService
@@ -8,4 +8,5 @@ __all__ = [
     "BaseService",
     "FileService",
     "EntityService",
+    "DatabaseService"
 ]

src/basic_memory/services/database_service.py

@@ -0,0 +1,158 @@
+"""Service for managing database lifecycle and schema validation."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Tuple, List
+
+from alembic.runtime.migration import MigrationContext
+from alembic.autogenerate import compare_metadata
+from loguru import logger
+from sqlalchemy import MetaData
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from basic_memory import db
+from basic_memory.config import ProjectConfig
+from basic_memory.models import Base
+
+
+async def check_schema_matches_models(session: AsyncSession) -> Tuple[bool, List[str]]:
+    """Check if database schema matches SQLAlchemy models.
+    
+    Returns:
+        tuple[bool, list[str]]: (matches, list of differences)
+    """
+    # Get current DB schema via migration context
+    conn = await session.connection()
+
+    def _compare_schemas(connection):
+        context = MigrationContext.configure(connection)
+        return compare_metadata(context, Base.metadata)
+
+    # Run comparison in sync context
+    differences = await conn.run_sync(_compare_schemas)
+    
+    if not differences:
+        return True, []
+        
+    # Format differences into readable messages
+    diff_messages = []
+    for diff in differences:
+        if diff[0] == 'add_table':
+            diff_messages.append(f"Missing table: {diff[1].name}")
+        elif diff[0] == 'remove_table':
+            diff_messages.append(f"Extra table: {diff[1].name}")
+        elif diff[0] == 'add_column':
+            diff_messages.append(f"Missing column: {diff[3]} in table {diff[2]}")
+        elif diff[0] == 'remove_column':
+            diff_messages.append(f"Extra column: {diff[3]} in table {diff[2]}")
+        elif diff[0] == 'modify_type':
+            diff_messages.append(f"Column type mismatch: {diff[3]} in table {diff[2]}")
+            
+    return False, diff_messages
+
+
+class DatabaseService:
+    """Manages database lifecycle including schema validation and backups."""
+
+    def __init__(
+        self,
+        config: ProjectConfig,
+        db_type: db.DatabaseType = db.DatabaseType.FILESYSTEM,
+    ):
+        self.config = config
+        self.db_path = Path(config.database_path)
+        self.db_type = db_type
+
+    async def create_backup(self) -> Optional[Path]:
+        """Create backup of existing database file.
+
+        Returns:
+            Optional[Path]: Path to backup file if created, None if no DB exists
+        """
+        if self.db_type == db.DatabaseType.MEMORY:
+            return None  # Skip backups for in-memory DB
+
+        if not self.db_path.exists():
+            return None
+
+        # Create backup with timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        backup_path = self.db_path.with_suffix(f".{timestamp}.backup")
+
+        try:
+            self.db_path.rename(backup_path)
+            logger.info(f"Created database backup: {backup_path}")
+            
+            # make a new empty file
+            self.db_path.touch()
+            return backup_path
+        except Exception as e:
+            logger.error(f"Failed to create database backup: {e}")
+            return None
+
+    async def initialize_db(self):
+        """Initialize database with current schema."""
+        logger.info("Initializing database...")
+
+        if self.db_type == db.DatabaseType.FILESYSTEM:
+            await self.create_backup()
+
+        # Drop existing tables if any
+        await db.drop_db()
+
+        # Create tables with current schema
+        await db.get_or_create_db(
+            db_path=self.db_path,
+            db_type=self.db_type
+        )
+
+        logger.info("Database initialized with current schema")
+
+    async def check_db(self) -> bool:
+        """Check database state and rebuild if schema doesn't match models.
+        
+        Returns:
+            bool: True if DB is ready for use, False if initialization failed
+        """
+        try:
+            _, session_maker = await db.get_or_create_db(
+                db_path=self.db_path,
+                db_type=self.db_type
+            )
+            async with db.scoped_session(session_maker) as db_session:
+                # Check actual schema matches
+                matches, differences = await check_schema_matches_models(db_session)
+                if not matches:
+                    logger.warning("Database schema does not match models:")
+                    for diff in differences:
+                        logger.warning(f"  {diff}")
+                    logger.info("Rebuilding database to match current models...")
+                    await self.initialize_db()
+                    return True
+                    
+                logger.info("Database schema matches models")
+                return True
+
+        except Exception as e:
+            logger.error(f"Database initialization failed: {e}")
+            return False
+
+    async def cleanup_backups(self, keep_count: int = 5):
+        """Clean up old database backups, keeping the N most recent."""
+        if self.db_type == db.DatabaseType.MEMORY:
+            return  # Skip cleanup for in-memory DB
+
+        backup_pattern = "*.backup"  # Use relative pattern
+        backups = sorted(
+            self.db_path.parent.glob(backup_pattern),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+
+        # Remove old backups
+        for backup in backups[keep_count:]:
+            try:
+                backup.unlink()
+                logger.debug(f"Removed old backup: {backup}")
+            except Exception as e:
+                logger.error(f"Failed to remove backup {backup}: {e}")

src/basic_memory/services/entity_service.py

@@ -49,20 +49,31 @@ class EntityService(BaseService[EntityModel]):
         """Get or generate unique permalink for an entity.
 
         Priority:
-        1. Use explicit permalink from markdown frontmatter if present
-        2. For existing files, keep current permalink
-        3. Generate new unique permalink for new files
+        1. If markdown has permalink and it's not used by another file -> use as is
+        2. If markdown has permalink but it's used by another file -> make unique
+        3. For existing files, keep current permalink from db
+        4. Generate new unique permalink from file path
         """
-        # If markdown has explicit permalink, try to use it
+        file_path = str(file_path)
+
+        # If markdown has explicit permalink, try to validate it
+        if markdown and markdown.frontmatter.permalink:
+            desired_permalink = markdown.frontmatter.permalink
+            existing = await self.repository.get_by_permalink(desired_permalink)
+
+            # If no conflict or it's our own file, use as is
+            if not existing or existing.file_path == file_path:
+                return desired_permalink
+
+        # For existing files, try to find current permalink
+        existing = await self.repository.get_by_file_path(file_path)
+        if existing:
+            return existing.permalink
+
+        # New file - generate permalink
         if markdown and markdown.frontmatter.permalink:
             desired_permalink = markdown.frontmatter.permalink
         else:
-            # For existing files, try to find current permalink
-            existing = await self.repository.get_by_file_path(str(file_path))
-            if existing:
-                return existing.permalink
-
-            # New file - generate permalink
             desired_permalink = generate_permalink(file_path)
 
         # Make unique if needed
@@ -72,7 +83,7 @@ class EntityService(BaseService[EntityModel]):
             permalink = f"{desired_permalink}-{suffix}"
             suffix += 1
             logger.debug(f"creating unique permalink: {permalink}")
-            
+
         return permalink
     
     async def create_or_update_entity(self, schema: EntitySchema) -> (EntityModel, bool):

src/basic_memory/sync/sync_service.py

@@ -5,6 +5,7 @@ from typing import Dict
 
 from loguru import logger
 
+from basic_memory import file_utils
 from basic_memory.markdown import EntityParser, EntityMarkdown
 from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService
@@ -100,13 +101,17 @@ class SyncService:
             )
 
             if permalink != entity_markdown.frontmatter.permalink:
-                # Permalink changed - update markdown and rewrite file
-                entity_markdown.frontmatter.metadata["permalink"] = permalink
-
-                # update file
+                # Add/update permalink in frontmatter
                 logger.info(f"Adding permalink '{permalink}' to file: {file_path}")
-                updated_checksum = await self.entity_service.file_service.markdown_processor.write_file(
-                    directory / file_path, entity_markdown)
+
+                # update markdown
+                entity_markdown.frontmatter.metadata["permalink"] = permalink
+                
+                # update file frontmatter
+                updated_checksum = await file_utils.update_frontmatter(
+                    directory / file_path,
+                    {"permalink": permalink}
+                )
 
                 # Update checksum in changes report since file was modified
                 changes.checksums[file_path] = updated_checksum

tests/conftest.py

@@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker
 
 from basic_memory import db
 from basic_memory.config import ProjectConfig
-from basic_memory.db import DatabaseType
+from basic_memory.db import DatabaseType, init_db
 from basic_memory.markdown import EntityParser
 from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.models import Base
@@ -21,7 +21,8 @@ from basic_memory.repository.observation_repository import ObservationRepository
 from basic_memory.repository.relation_repository import RelationRepository
 from basic_memory.repository.search_repository import SearchRepository
 from basic_memory.services import (
-    EntityService,
+EntityService,
+DatabaseService,
 )
 from basic_memory.services.file_service import FileService
 from basic_memory.services.link_resolver import LinkResolver
@@ -392,4 +393,3 @@ def watch_service(sync_service, file_service, test_config):
         file_service=file_service,
         config=test_config
     )
-

tests/services/test_database_service.py

@@ -0,0 +1,158 @@
+"""Tests for DatabaseService."""
+
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import AsyncGenerator
+
+import pytest
+import pytest_asyncio
+from sqlalchemy import Column, String, Table, text
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, AsyncSession
+
+from basic_memory import db
+from basic_memory.config import ProjectConfig
+from basic_memory.db import DatabaseType
+from basic_memory.models import Base
+from basic_memory.services.database_service import DatabaseService
+from basic_memory.sync import SyncService
+
+
+@pytest_asyncio.fixture(scope="function")
+async def engine_factory(
+    test_config,
+) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
+    """Special version of the engine factory fixture that uses a FILESYSTEM db_type"""
+    async with db.engine_session_factory(
+        db_path=test_config.database_path, db_type=DatabaseType.FILESYSTEM
+    ) as (engine, session_maker):
+        # Initialize database
+        async with db.scoped_session(session_maker) as session:
+            await session.execute(text("PRAGMA foreign_keys=ON"))
+            conn = await session.connection()
+            await conn.run_sync(Base.metadata.create_all)
+
+        yield engine, session_maker
+
+@pytest_asyncio.fixture
+async def database_service(
+    test_config: ProjectConfig,
+    sync_service: SyncService,
+) -> DatabaseService:
+    """Create DatabaseManagementService instance for testing."""
+    return DatabaseService(
+        config=test_config,
+        db_type = DatabaseType.FILESYSTEM
+    )
+
+@pytest.mark.asyncio
+async def test_check_db_initializes_new_db(
+    database_service: DatabaseService,
+):
+    """Test that check_db initializes new database."""
+    # Ensure DB doesn't exist
+    if Path(database_service.db_path).exists():
+        Path(database_service.db_path).unlink()
+
+    # Check DB - should initialize
+    assert await database_service.check_db()
+
+
+@pytest.mark.asyncio
+async def test_check_db_rebuilds_on_schema_mismatch(
+    database_service: DatabaseService,
+    session_maker,
+):
+    """Test that check_db rebuilds DB when schema doesn't match."""
+    # Initialize DB first
+    assert await database_service.check_db()
+
+    # Alter an existing table to remove a column
+    async with db.scoped_session(session_maker) as session:
+        conn = await session.connection()
+        # Create temp table
+        await conn.execute(text("""
+            CREATE TABLE entity_temp (
+                id INTEGER PRIMARY KEY,
+                title TEXT,
+                entity_type TEXT,
+                content_type TEXT,
+                permalink TEXT,
+                file_path TEXT,
+                checksum TEXT,
+                created_at TIMESTAMP,
+                updated_at TIMESTAMP
+                -- Deliberately omit entity_metadata column
+            )
+        """))
+        # Drop original table
+        await conn.execute(text("DROP TABLE entity"))
+        # Rename temp table
+        await conn.execute(text("ALTER TABLE entity_temp RENAME TO entity"))
+        await session.commit()
+
+    # Check DB - should detect missing column and rebuild
+    assert await database_service.check_db()
+
+    # Verify entity_metadata column exists now
+    async with db.scoped_session(session_maker) as session:
+        result = await session.execute(text("""
+            SELECT sql FROM sqlite_master 
+            WHERE type='table' AND name='entity'
+        """))
+        create_sql = result.scalar()
+        assert 'entity_metadata' in create_sql.lower()
+
+@pytest.mark.asyncio
+async def test_backup_creates_timestamped_file(
+    database_service: DatabaseService,
+):
+    """Test that backup creates properly named backup file."""
+    if database_service.db_type == db.DatabaseType.MEMORY:
+        return
+
+    # Create dummy DB file
+    database_service.db_path.parent.mkdir(parents=True, exist_ok=True)
+    database_service.db_path.write_text("test content")
+
+    # Create backup
+    backup_path = await database_service.create_backup()
+    
+    assert backup_path is not None
+    assert backup_path.exists()
+    assert backup_path.suffix == ".backup"
+    assert datetime.now().strftime("%Y%m%d") in backup_path.name
+
+
+@pytest.mark.asyncio
+async def test_cleanup_backups_keeps_recent(
+    database_service: DatabaseService,
+):
+    """Test that cleanup_backups keeps N most recent backups."""
+    if database_service.db_type == db.DatabaseType.MEMORY:
+        return
+
+    # Create backup directory
+    backup_dir = database_service.db_path.parent
+    backup_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create some test backup files with different timestamps
+    backup_times = [
+        datetime.now() - timedelta(days=i)
+        for i in range(7)  # Create 7 backups
+    ]
+    
+    for dt in backup_times:
+        timestamp = dt.strftime("%Y%m%d_%H%M%S")
+        backup_path = database_service.db_path.with_suffix(f".{timestamp}.backup")
+        backup_path.write_text("test backup")
+        # Set mtime to match our timestamp
+        backup_path.touch()
+        ts = dt.timestamp()
+
+    # Clean up keeping 5 most recent
+    await database_service.cleanup_backups(keep_count=5)
+
+    # Check that we have exactly 5 backups left
+    backup_pattern = "*.backup"
+    remaining = list(backup_dir.glob(backup_pattern))
+    assert len(remaining) == 5

tests/sync/test_sync_service.py

@@ -777,5 +777,70 @@ test content
     await sync_service.sync(test_config.home)
 
     # Check permalinks
-    file_one_content, _ = await file_service.read_file(two_file)
-    assert "permalink: one-1" in file_one_content
+    file_two_content, _ = await file_service.read_file(two_file)
+    assert "permalink: two" in file_two_content
+
+    # new content with duplicate permalink
+    new_content = """
+---
+title: new.md
+type: note
+permalink: one
+tags: []
+---
+
+test content
+"""
+    new_file = project_dir / "new.md"
+    await create_test_file(new_file)
+
+    # Run another time 
+    await sync_service.sync(test_config.home)
+
+    # Should still have same permalink
+    new_file_content, _ = await file_service.read_file(new_file)
+    assert "permalink: new" in new_file_content
+
+    
+
+
+@pytest.mark.asyncio
+async def test_sync_duplicate_observations(
+        sync_service: SyncService,
+        test_config: ProjectConfig,
+        file_service: FileService,
+):
+    """Test that sync resolves permalink conflicts on update."""
+    project_dir = test_config.home
+
+    content = """
+---
+title: a note
+type: note
+tags: []
+---
+
+test content
+
+- [note] one observation
+"""
+
+    note_file = project_dir / "note.md"
+    await create_test_file(note_file, content)
+
+    # Run sync
+    await sync_service.sync(test_config.home)
+
+    # Check permalinks
+    file_one_content, _ = await file_service.read_file(note_file)
+    assert """---
+title: a note
+type: note
+tags: []
+permalink: note
+---
+
+test content
+
+- [note] one observation
+""".strip() == file_one_content

tests/utils/test_file_utils.py

@@ -8,14 +8,12 @@ from basic_memory.file_utils import (
     compute_checksum,
     ensure_directory,
     write_file_atomic,
-    add_frontmatter,
     parse_frontmatter,
     has_frontmatter,
     remove_frontmatter,
-    parse_content_with_frontmatter,
     FileError,
     FileWriteError,
-    ParseError,
+    ParseError, update_frontmatter,
 )
 
 
@@ -69,26 +67,6 @@ async def test_write_file_atomic_error(tmp_path: Path):
         await write_file_atomic(test_file, "test content")
 
 
-@pytest.mark.asyncio
-async def test_add_frontmatter():
-    """Test adding frontmatter."""
-    content = "test content"
-    metadata = {"title": "Test", "tags": ["a", "b"]}
-
-    result = await add_frontmatter(content, metadata)
-
-    # Should have frontmatter delimiters
-    assert result.startswith("---\n")
-    assert "---\n\n" in result
-
-    # Should include metadata
-    assert "title: Test" in result
-    assert "- a\n- b" in result or "['a', 'b']" in result
-
-    # Should preserve content
-    assert result.endswith("test content")
-
-
 def test_has_frontmatter():
     """Test frontmatter detection."""
     # Valid frontmatter
@@ -186,69 +164,72 @@ content""")
 
 
 @pytest.mark.asyncio
-async def test_parse_content_with_frontmatter():
-    """Test combined frontmatter and content parsing."""
-    # Full document
-    content = """---
-title: Test
-tags:
-  - a
-  - b
----
-test content"""
+async def test_update_frontmatter(tmp_path: Path):
+    """Test updating frontmatter in a file."""
+    test_file = tmp_path / "test.md"
 
-    frontmatter, body = await parse_content_with_frontmatter(content)
-    assert frontmatter == {"title": "Test", "tags": ["a", "b"]}
-    assert body == "test content"
+    # Test 1: Add frontmatter to file without any
+    content = "# Test Content\n\nSome content here"
+    test_file.write_text(content)
 
-    # No frontmatter
-    content = "test content"
-    frontmatter, body = await parse_content_with_frontmatter(content)
-    assert frontmatter == {}
-    assert body == "test content"
+    updates = {"title": "Test", "type": "note"}
+    checksum = await update_frontmatter(test_file, updates)
 
-    # Empty document
-    frontmatter, body = await parse_content_with_frontmatter("")
-    assert frontmatter == {}
-    assert body == ""
+    # Verify content
+    updated = test_file.read_text()
+    assert "title: Test" in updated
+    assert "type: note" in updated
+    assert "Test Content" in updated
+    assert "Some content here" in updated
 
-    # Only frontmatter
-    content = """---
-title: Test
----
-"""
-    frontmatter, body = await parse_content_with_frontmatter(content)
-    assert frontmatter == {"title": "Test"}
-    assert body == ""
+    # Verify structure
+    fm = parse_frontmatter(updated)
+    assert fm == updates
+    assert remove_frontmatter(updated).strip() == content
+
+    # Test 2: Update existing frontmatter
+    updates = {"type": "doc", "tags": ["test"]}
+    new_checksum = await update_frontmatter(test_file, updates)
+
+    # Verify checksum changed
+    assert new_checksum != checksum
+
+    # Verify content
+    updated = test_file.read_text()
+    fm = parse_frontmatter(updated)
+    assert fm == {"title": "Test", "type": "doc", "tags": ["test"]}
+    assert "Test Content" in updated
+
+    # Test 3: Update with empty dict shouldn't change anything
+    checksum_before = await compute_checksum(test_file.read_text())
+    new_checksum = await update_frontmatter(test_file, {})
+    assert new_checksum == checksum_before
+
+    # Test 4: Handle multi-line content properly
+    content = """# Heading
+
+Some content
+
+## Section
+- Point 1
+- Point 2
+
+### Subsection
+More content here"""
+
+    test_file.write_text(content)
+    await update_frontmatter(test_file, {"title": "Test"})
+
+    updated = test_file.read_text()
+    assert remove_frontmatter(updated).strip() == content
 
 
 @pytest.mark.asyncio
-async def test_frontmatter_whitespace_handling():
-    """Test frontmatter handling with various whitespace."""
-    # Extra newlines before frontmatter
-    content = """
+async def test_update_frontmatter_errors(tmp_path: Path):
+    """Test error handling in update_frontmatter."""
+    test_file = tmp_path / "test.md"
 
----
-title: Test
----
-content"""
-    assert has_frontmatter(content.strip())
-    frontmatter = parse_frontmatter(content.strip())
-    assert frontmatter == {"title": "Test"}
-
-    # Extra newlines after frontmatter
-    content = """---
-title: Test
----
-
-
-content"""
-    result = await add_frontmatter("content", {"title": "Test"})
-    assert result.count("\n\n") == 1  # Should normalize to single blank line
-
-    # Spaces around content
-    content = """---
-title: Test
----
-   content   """
-    assert remove_frontmatter(content).strip() == "content"
+    # Test 1: Invalid file path
+    nonexistent = tmp_path / "nonexistent" / "test.md"
+    with pytest.raises(FileError):
+        await update_frontmatter(nonexistent, {"title": "Test"})

uv.lock

@@ -13,6 +13,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/00/c4/c93eb22025a2de6b83263dfe3d7df2e19138e345bca6f18dba7394120930/aiosqlite-0.20.0-py3-none-any.whl", hash = "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6", size = 15564 },
 ]
 
+[[package]]
+name = "alembic"
+version = "1.14.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mako" },
+    { name = "sqlalchemy" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/99/09/f844822e4e847a3f0bd41797f93c4674cd4d2462a3f6c459aa528cdf786e/alembic-1.14.1.tar.gz", hash = "sha256:496e888245a53adf1498fcab31713a469c65836f8de76e01399aa1c3e90dd213", size = 1918219 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/54/7e/ac0991d1745f7d755fc1cd381b3990a45b404b4d008fc75e2a983516fbfe/alembic-1.14.1-py3-none-any.whl", hash = "sha256:1acdd7a3a478e208b0503cd73614d5e4c6efafa4e73518bb60e4f2846a37b1c5", size = 233565 },
+]
+
 [[package]]
 name = "annotated-types"
 version = "0.7.0"
@@ -51,6 +65,7 @@ version = "0.0.1"
 source = { editable = "." }
 dependencies = [
     { name = "aiosqlite" },
+    { name = "alembic" },
     { name = "dateparser" },
     { name = "fastapi", extra = ["standard"] },
     { name = "greenlet" },
@@ -87,6 +102,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "aiosqlite", specifier = ">=0.20.0" },
+    { name = "alembic", specifier = ">=1.14.1" },
     { name = "dateparser", specifier = ">=1.2.0" },
     { name = "fastapi", extras = ["standard"], specifier = ">=0.115.8" },
     { name = "greenlet", specifier = ">=3.1.1" },
@@ -430,6 +446,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595 },
 ]
 
+[[package]]
+name = "mako"
+version = "1.3.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/62/4f/ddb1965901bc388958db9f0c991255b2c469349a741ae8c9cd8a562d70a6/mako-1.3.9.tar.gz", hash = "sha256:b5d65ff3462870feec922dbccf38f6efb44e5714d7b593a656be86663d8600ac", size = 392195 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cd/83/de0a49e7de540513f53ab5d2e105321dedeb08a8f5850f0208decf4390ec/Mako-1.3.9-py3-none-any.whl", hash = "sha256:95920acccb578427a9aa38e37a186b1e43156c87260d7ba18ca63aa4c7cbd3a1", size = 78456 },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"