mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Merge branch 'document-store' into document-entity-write
This commit is contained in:
phernandez
@@ -36,5 +36,6 @@ projects/*.db
|
||||
projects/*.db-journal
|
||||
/.coverage
|
||||
|
||||
**/.DS_Store
|
||||
|
||||
*.log
|
||||
@@ -1,129 +0,0 @@
|
||||
# basic-memory
|
||||
|
||||
Local-first knowledge management system that combines Zettelkasten methodology with knowledge graphs. Built using SQLite and markdown files, it enables seamless capture and connection of ideas while maintaining user control over data.
|
||||
|
||||
## Features
|
||||
|
||||
- Local-first design using SQLite and markdown files
|
||||
- Combines Zettelkasten principles with knowledge graph capabilities
|
||||
- Everything readable/writable as markdown
|
||||
- Project isolation for focused context
|
||||
- Rich querying and traversal through SQLite index
|
||||
- Built with Python 3.12, SQLAlchemy, and modern tooling
|
||||
|
||||
## Development
|
||||
|
||||
Setup your development environment:
|
||||
|
||||
```bash
|
||||
# Create and activate virtual environment
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate # or `.venv/Scripts/activate` on Windows
|
||||
|
||||
# Install dependencies including dev tools
|
||||
pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
Run tests:
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0-or-later
|
||||
|
||||
|
||||
project info memory store
|
||||
~/.npm/_npx/15b07286cbcc3329/node_modules/@modelcontextprotocol/server-memory/dist/memory.json
|
||||
|
||||
|
||||
## Running locally
|
||||
|
||||
See: https://modelcontextprotocol.io/docs/first-server/python#connect-to-claude-desktop
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"basic-memory": {
|
||||
"command": "uv",
|
||||
"args": [
|
||||
"--directory",
|
||||
"/Users/phernandez/dev/basicmachines/basic-memory",
|
||||
"run",
|
||||
"src/basic_memory/mcp/server.py"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### logs
|
||||
|
||||
/Users/phernandez/Library/Logs/Claude/mcp-server-basic-memory.log
|
||||
|
||||
|
||||
|
||||
|
||||
# MCP quickstart
|
||||
https://modelcontextprotocol.io/quickstart
|
||||
|
||||
Claude Desktop config json
|
||||
/Users/phernandez/Library/Application Support/Claude/claude_desktop_config.json
|
||||
|
||||
## stdout logging
|
||||
|
||||
logs-dir:
|
||||
/Users/phernandez/Library/Logs/Claude
|
||||
|
||||
- mcp-server-sqlite.log
|
||||
- mcp.log
|
||||
|
||||
To test changes efficiently:
|
||||
|
||||
- Configuration changes: Restart Claude Desktop
|
||||
- Server code changes: Use Command-R to reload
|
||||
- Quick iteration: Use Inspector during development
|
||||
- https://modelcontextprotocol.io/docs/tools/inspector
|
||||
|
||||
eg.
|
||||
```bash
|
||||
npx @modelcontextprotocol/inspector uvx mcp-server-sqlite --db-path /Users/phernandez/dev/basicmachines/mcp-quickstart/test.db
|
||||
```
|
||||
|
||||
## tools
|
||||
Inspector: https://modelcontextprotocol.io/docs/tools/inspector
|
||||
Debugger: https://modelcontextprotocol.io/docs/tools/debugging
|
||||
|
||||
## dev tools
|
||||
|
||||
```bash
|
||||
jq '.allowDevTools = true' ~/Library/Application\ Support/Claude/developer_settings.json > tmp.json \
|
||||
&& mv tmp.json ~/Library/Application\ Support/Claude/developer_settings.json
|
||||
|
||||
```
|
||||
|
||||
Open DevTools: Command-Option-Shift-i
|
||||
|
||||
|
||||
## TODO
|
||||
|
||||
logs
|
||||
|
||||
/Users/phernandez/Library/Logs/Claude/mcp-server-basic-memory.log
|
||||
|
||||
```text
|
||||
2024-12-08 20:32:38.802 | INFO | __main__:run_server:264 - Starting MCP server basic-memory
|
||||
The garbage collector is trying to clean up non-checked-in connection <AdaptedConnection <Connection(Thread-2, started daemon 6227046400)>>, which will be dropped, as it cannot be safely terminated. Please ensure that SQLAlchemy pooled connections are returned to the pool explicitly, either by calling ``close()`` or by using appropriate context managers to manage their lifecycle.
|
||||
The garbage collector is trying to clean up non-checked-in connection <AdaptedConnection <Connection(Thread-3, started daemon 6243872768)>>, which will be dropped, as it cannot be safely terminated. Please ensure that SQLAlchemy pooled connections are returned to the pool explicitly, either by calling ``close()`` or by using appropriate context managers to manage their lifecycle.
|
||||
~
|
||||
```
|
||||
|
||||
more info about setting log level: https://modelcontextprotocol.io/docs/first-server/python
|
||||
- scroll down to logging
|
||||
|
||||
|
||||
```bash
|
||||
basic-memory migrate json /Users/phernandez/.npm/_npx/15b07286cbcc3329/node_modules/@modelcontextprotocol/server-memory/dist/memory.json /Users/phernandez/.basic-memory/projects/default
|
||||
```
|
||||
|
||||
|
||||
+3
-1
@@ -19,6 +19,8 @@ dependencies = [
|
||||
"loguru>=0.7.3",
|
||||
"pyright>=1.1.390",
|
||||
"basic-foundation",
|
||||
"markdown-it-py>=3.0.0",
|
||||
"python-frontmatter>=1.1.0"
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -75,4 +77,4 @@ ignore = [
|
||||
defineConstant = { DEBUG = true }
|
||||
reportMissingImports = "error"
|
||||
reportMissingTypeStubs = false
|
||||
pythonVersion = "3.12"
|
||||
pythonVersion = "3.12"
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Models package for basic-memory."""
|
||||
|
||||
from basic_memory.models.base import Base
|
||||
from basic_memory.models.documents import Document
|
||||
from basic_memory.models.knowledge import Entity, Observation, Relation
|
||||
|
||||
__all__ = [
|
||||
'Base',
|
||||
'Document',
|
||||
'Entity',
|
||||
'Observation',
|
||||
'Relation'
|
||||
]
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Base model class for SQLAlchemy models."""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncAttrs
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(AsyncAttrs, DeclarativeBase):
|
||||
"""Base class for all models"""
|
||||
pass
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Document model for tracking files in the knowledge base."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
from sqlalchemy import String, DateTime, text, JSON
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from basic_memory.models.base import Base
|
||||
|
||||
|
||||
class Document(Base):
|
||||
"""
|
||||
Tracks documents in the filesystem.
|
||||
|
||||
Documents are the source of truth for content, while this table
|
||||
provides indexing and metadata storage. Like git, the filesystem
|
||||
is the real source of truth.
|
||||
"""
|
||||
|
||||
__tablename__ = "documents"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String, unique=True, nullable=False)
|
||||
checksum: Mapped[str] = mapped_column(String, nullable=False)
|
||||
doc_metadata: Mapped[Optional[dict]] = mapped_column(
|
||||
JSON, nullable=True
|
||||
) # renamed from metadata
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP"))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP")
|
||||
)
|
||||
|
||||
# Relationships
|
||||
entities: Mapped[List["Entity"]] = relationship( # pyright: ignore [reportUndefinedVariable] # noqa: F821
|
||||
"Entity", back_populates="document", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Document(id={self.id}, path='{self.path}')"
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Knowledge graph models for basic-memory."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
String, DateTime, ForeignKey, Text, Integer,
|
||||
text, UniqueConstraint
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from basic_memory.models.base import Base
|
||||
from basic_memory.utils import sanitize_name
|
||||
|
||||
|
||||
class Entity(Base):
|
||||
"""
|
||||
Core entity in the knowledge graph.
|
||||
|
||||
Entities are the primary nodes in the knowledge graph. Each entity has:
|
||||
- A unique identifier (text, based on type/name path)
|
||||
- A name
|
||||
- An entity type (e.g., "person", "organization", "event")
|
||||
- A description (optional)
|
||||
- A list of observations
|
||||
"""
|
||||
__tablename__ = "entity"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("entity_type", "name", name="uix_entity_type_name"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String)
|
||||
entity_type: Mapped[str] = mapped_column(String)
|
||||
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
server_default=text("CURRENT_TIMESTAMP")
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
server_default=text("CURRENT_TIMESTAMP"),
|
||||
onupdate=text("CURRENT_TIMESTAMP")
|
||||
)
|
||||
|
||||
# Link to source document
|
||||
doc_id: Mapped[Optional[int]] = mapped_column(
|
||||
Integer,
|
||||
ForeignKey("documents.id", ondelete="SET NULL"),
|
||||
nullable=True
|
||||
)
|
||||
|
||||
# Relationships
|
||||
document: Mapped["Document"] = relationship(
|
||||
"Document",
|
||||
back_populates="entities"
|
||||
)
|
||||
observations: Mapped[List["Observation"]] = relationship(
|
||||
"Observation",
|
||||
back_populates="entity",
|
||||
cascade="all, delete-orphan"
|
||||
)
|
||||
outgoing_relations: Mapped[List["Relation"]] = relationship(
|
||||
"Relation",
|
||||
foreign_keys="[Relation.from_id]",
|
||||
back_populates="from_entity",
|
||||
cascade="all, delete-orphan"
|
||||
)
|
||||
incoming_relations: Mapped[List["Relation"]] = relationship(
|
||||
"Relation",
|
||||
foreign_keys="[Relation.to_id]",
|
||||
back_populates="to_entity",
|
||||
cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
@property
|
||||
def relations(self):
|
||||
return self.outgoing_relations + self.incoming_relations
|
||||
|
||||
@classmethod
|
||||
def generate_id(cls, entity_type: str, name: str) -> str:
|
||||
"""Generate a filesystem path-based ID for this entity."""
|
||||
# Use common normalization for filesystem safety
|
||||
safe_name = sanitize_name(name)
|
||||
return f"{entity_type}/{safe_name}"
|
||||
|
||||
def get_file_path(self) -> str:
|
||||
"""Get the filesystem path for this entity."""
|
||||
return f"{self.id}.md" # id is already in path format
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Entity(id='{self.id}', name='{self.name}', type='{self.entity_type}')"
|
||||
|
||||
|
||||
class Observation(Base):
|
||||
"""
|
||||
Observations are discrete pieces of information about an entity. They are:
|
||||
- Stored as strings
|
||||
- Attached to specific entities
|
||||
- Can be added or removed independently
|
||||
- Should be atomic (one fact per observation)
|
||||
"""
|
||||
__tablename__ = "observation"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
entity_id: Mapped[str] = mapped_column(
|
||||
String,
|
||||
ForeignKey("entity.id", ondelete="CASCADE"),
|
||||
index=True
|
||||
)
|
||||
content: Mapped[str] = mapped_column(String)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
server_default=text("CURRENT_TIMESTAMP")
|
||||
)
|
||||
context: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
|
||||
# Relationships
|
||||
entity: Mapped[Entity] = relationship("Entity", back_populates="observations")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
content = self.content[:50] + "..." if len(self.content) > 50 else self.content
|
||||
return f"Observation(id={self.id}, entity='{self.entity_id}', content='{content}')"
|
||||
|
||||
|
||||
class Relation(Base):
|
||||
"""
|
||||
Relations define directed connections between entities.
|
||||
They are always stored in active voice and describe how entities
|
||||
interact or relate to each other.
|
||||
"""
|
||||
__tablename__ = "relation"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
from_id: Mapped[str] = mapped_column(
|
||||
String,
|
||||
ForeignKey("entity.id", ondelete="CASCADE"),
|
||||
index=True
|
||||
)
|
||||
to_id: Mapped[str] = mapped_column(
|
||||
String,
|
||||
ForeignKey("entity.id", ondelete="CASCADE"),
|
||||
index=True
|
||||
)
|
||||
relation_type: Mapped[str] = mapped_column(String)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime,
|
||||
server_default=text("CURRENT_TIMESTAMP")
|
||||
)
|
||||
context: Mapped[Optional[str]] = mapped_column(String, nullable=True)
|
||||
|
||||
# Relationships
|
||||
from_entity: Mapped[Entity] = relationship(
|
||||
"Entity",
|
||||
foreign_keys=[from_id],
|
||||
back_populates="outgoing_relations"
|
||||
)
|
||||
to_entity: Mapped[Entity] = relationship(
|
||||
"Entity",
|
||||
foreign_keys=[to_id],
|
||||
back_populates="incoming_relations"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Relation(id={self.id}, from='{self.from_id}', type='{self.relation_type}', to='{self.to_id}')"
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Repository for document operations."""
|
||||
|
||||
from typing import Optional, Sequence, List
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession
|
||||
|
||||
from basic_memory.models import Document
|
||||
from basic_memory.repository.repository import Repository
|
||||
|
||||
|
||||
class DocumentRepository(Repository[Document]):
|
||||
"""Repository for managing documents in the database."""
|
||||
|
||||
def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
|
||||
super().__init__(session_maker, Document)
|
||||
|
||||
async def find_by_path(self, path: str) -> Optional[Document]:
|
||||
"""Find a document by its path."""
|
||||
query = select(Document).where(Document.path == path)
|
||||
return await self.find_one(query)
|
||||
|
||||
async def find_by_checksum(self, checksum: str) -> Sequence[Document]:
|
||||
"""Find all documents with a given checksum."""
|
||||
async with self.session_maker() as session:
|
||||
result = await session.execute(
|
||||
select(Document).where(Document.checksum == checksum)
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
async def find_changed(self, checksums: dict[str, str]) -> List[Document]:
|
||||
"""
|
||||
Find documents that have changed based on their checksums.
|
||||
|
||||
Args:
|
||||
checksums: Dict mapping paths to their current checksums
|
||||
|
||||
Returns:
|
||||
List of documents whose checksums don't match (excluding untracked files)
|
||||
"""
|
||||
changed = []
|
||||
for path, checksum in checksums.items():
|
||||
doc = await self.find_by_path(path)
|
||||
if doc and doc.checksum != checksum: # Only include tracked files that changed
|
||||
changed.append(doc)
|
||||
return changed
|
||||
@@ -1,33 +1,15 @@
|
||||
"""Service layer exceptions and imports."""
|
||||
"""Services package."""
|
||||
|
||||
|
||||
class ServiceError(Exception):
|
||||
"""Base exception for service errors"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseSyncError(ServiceError):
|
||||
"""Raised when database sync fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RelationError(ServiceError):
|
||||
"""Base exception for relation-specific errors"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
from .entity_service import EntityService
|
||||
from .observation_service import ObservationService
|
||||
from .relation_service import RelationService
|
||||
from basic_memory.services.document_service import DocumentService
|
||||
from basic_memory.services.entity_service import EntityService
|
||||
from basic_memory.services.observation_service import ObservationService
|
||||
from basic_memory.services.relation_service import RelationService
|
||||
from basic_memory.services.service import BaseService
|
||||
|
||||
__all__ = [
|
||||
"ServiceError",
|
||||
"DatabaseSyncError",
|
||||
"RelationError",
|
||||
"EntityService",
|
||||
"ObservationService",
|
||||
"RelationService",
|
||||
]
|
||||
'BaseService',
|
||||
'DocumentService',
|
||||
'EntityService',
|
||||
'ObservationService',
|
||||
'RelationService',
|
||||
]
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Service for managing documents in the system."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.models import Document
|
||||
from basic_memory.repository.document_repository import DocumentRepository
|
||||
from basic_memory.services.service import BaseService
|
||||
|
||||
|
||||
class DocumentError(Exception):
|
||||
"""Base exception for document operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DocumentNotFoundError(DocumentError):
|
||||
"""Raised when a document doesn't exist."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DocumentWriteError(DocumentError):
|
||||
"""Raised when document file operations fail."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DocumentService(BaseService[DocumentRepository]):
|
||||
"""
|
||||
Service for managing documents and their metadata.
|
||||
|
||||
Handles both file operations and database tracking, keeping
|
||||
them in sync. The filesystem is the source of truth.
|
||||
"""
|
||||
|
||||
def __init__(self, document_repository: DocumentRepository):
|
||||
super().__init__(document_repository)
|
||||
|
||||
async def compute_checksum(self, content: str) -> str:
|
||||
"""Compute SHA-256 checksum of content."""
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
async def ensure_parent_directory(self, path: Path) -> None:
|
||||
"""
|
||||
Ensure parent directory exists and is writable.
|
||||
|
||||
Args:
|
||||
path: Path to check
|
||||
|
||||
Raises:
|
||||
DocumentWriteError: If directory cannot be created or is not writable
|
||||
"""
|
||||
parent = path.parent
|
||||
try:
|
||||
if not parent.exists():
|
||||
parent.mkdir(parents=True)
|
||||
|
||||
# Verify we can write to it
|
||||
test_file = parent / ".write_test"
|
||||
test_file.touch()
|
||||
test_file.unlink()
|
||||
except Exception as e:
|
||||
raise DocumentWriteError(f"Directory not writable: {parent}: {e}")
|
||||
|
||||
async def list_documents(self) -> List[Document]:
|
||||
"""List all documents in the database."""
|
||||
return await self.repository.find_all()
|
||||
|
||||
async def create_document(
|
||||
self, path: str, content: str, metadata: Optional[Dict[str, Any]] = None
|
||||
) -> Document:
|
||||
"""
|
||||
Create a new document.
|
||||
|
||||
Args:
|
||||
path: Path where to create the document
|
||||
content: Document content
|
||||
metadata: Optional metadata to store
|
||||
|
||||
Returns:
|
||||
Created document record
|
||||
|
||||
Raises:
|
||||
DocumentWriteError: If file cannot be written
|
||||
"""
|
||||
logger.debug(f"Creating document at {path}")
|
||||
|
||||
# Ensure parent directories exist and are writable
|
||||
file_path = Path(path)
|
||||
await self.ensure_parent_directory(file_path)
|
||||
|
||||
# Write file first
|
||||
try:
|
||||
file_path.write_text(content)
|
||||
except Exception as e:
|
||||
raise DocumentWriteError(f"Failed to write document file: {e}")
|
||||
|
||||
# After file is written, create database record
|
||||
try:
|
||||
checksum = await self.compute_checksum(content)
|
||||
doc = await self.repository.create(
|
||||
{"path": str(path), "checksum": checksum, "doc_metadata": metadata}
|
||||
)
|
||||
return doc
|
||||
except Exception:
|
||||
# If database operation fails, clean up the file
|
||||
file_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
async def read_document(self, path: str) -> tuple[Document, str]:
|
||||
"""
|
||||
Read a document and its content.
|
||||
|
||||
Args:
|
||||
path: Path to the document
|
||||
|
||||
Returns:
|
||||
Tuple of (document record, content)
|
||||
|
||||
Raises:
|
||||
DocumentNotFoundError: If document doesn't exist
|
||||
"""
|
||||
logger.debug(f"Reading document at {path}")
|
||||
|
||||
# Check if file exists
|
||||
file_path = Path(path)
|
||||
if not file_path.exists():
|
||||
raise DocumentNotFoundError(f"Document not found: {path}")
|
||||
|
||||
# Read content first since file is source of truth
|
||||
try:
|
||||
content = file_path.read_text()
|
||||
except Exception as e:
|
||||
raise DocumentError(f"Failed to read document: {e}")
|
||||
|
||||
# Get document record
|
||||
doc = await self.repository.find_by_path(str(path))
|
||||
if not doc:
|
||||
# File exists but no DB record - create one
|
||||
checksum = await self.compute_checksum(content)
|
||||
doc = await self.repository.create({"path": str(path), "checksum": checksum})
|
||||
|
||||
return doc, content
|
||||
|
||||
async def update_document(
|
||||
self, path: str, content: str, metadata: Optional[Dict[str, Any]] = None
|
||||
) -> Document:
|
||||
"""
|
||||
Update an existing document.
|
||||
|
||||
Args:
|
||||
path: Path to the document
|
||||
content: New content
|
||||
metadata: Optional new metadata
|
||||
|
||||
Returns:
|
||||
Updated document record
|
||||
|
||||
Raises:
|
||||
DocumentNotFoundError: If document doesn't exist
|
||||
DocumentWriteError: If update fails
|
||||
"""
|
||||
logger.debug(f"Updating document at {path}")
|
||||
|
||||
# Verify file exists
|
||||
file_path = Path(path)
|
||||
if not file_path.exists():
|
||||
raise DocumentNotFoundError(f"Document not found: {path}")
|
||||
|
||||
# Write new content first
|
||||
try:
|
||||
file_path.write_text(content)
|
||||
except Exception as e:
|
||||
raise DocumentWriteError(f"Failed to write document: {e}")
|
||||
|
||||
# Update database record
|
||||
doc = await self.repository.find_by_path(str(path))
|
||||
if not doc:
|
||||
# File exists but no DB record - create one
|
||||
checksum = await self.compute_checksum(content)
|
||||
return await self.repository.create(
|
||||
{"path": str(path), "checksum": checksum, "doc_metadata": metadata}
|
||||
)
|
||||
|
||||
# Update existing record
|
||||
checksum = await self.compute_checksum(content)
|
||||
update_data = {"checksum": checksum}
|
||||
if metadata is not None:
|
||||
update_data["doc_metadata"] = metadata
|
||||
|
||||
updated_document = await self.repository.update(doc.id, update_data)
|
||||
assert updated_document is not None, f"Could not update document {doc.id}"
|
||||
return updated_document
|
||||
|
||||
async def delete_document(self, path: str) -> None:
|
||||
"""
|
||||
Delete a document.
|
||||
|
||||
Args:
|
||||
path: Path to the document
|
||||
|
||||
Raises:
|
||||
DocumentNotFoundError: If document doesn't exist
|
||||
DocumentWriteError: If deletion fails
|
||||
"""
|
||||
logger.debug(f"Deleting document at {path}")
|
||||
|
||||
# Verify file exists
|
||||
file_path = Path(path)
|
||||
if not file_path.exists():
|
||||
raise DocumentNotFoundError(f"Document not found: {path}")
|
||||
|
||||
# Delete file first
|
||||
try:
|
||||
file_path.unlink()
|
||||
except Exception as e:
|
||||
raise DocumentWriteError(f"Failed to delete document: {e}")
|
||||
|
||||
# Delete database record if it exists
|
||||
doc = await self.repository.find_by_path(str(path))
|
||||
if doc:
|
||||
await self.repository.delete(doc.id)
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Service for syncing files with the database."""
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Set
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.repository.document_repository import DocumentRepository
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncReport:
|
||||
"""Report of sync results."""
|
||||
new: Set[str]
|
||||
modified: Set[str]
|
||||
deleted: Set[str]
|
||||
|
||||
@property
|
||||
def total_changes(self) -> int:
|
||||
return len(self.new) + len(self.modified) + len(self.deleted)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"Changes detected:\n"
|
||||
f" New files: {len(self.new)}\n"
|
||||
f" Modified: {len(self.modified)}\n"
|
||||
f" Deleted: {len(self.deleted)}"
|
||||
)
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
"""Raised when sync operations fail."""
|
||||
pass
|
||||
|
||||
|
||||
class FileSyncService:
|
||||
"""Service for keeping files and database in sync."""
|
||||
|
||||
def __init__(self, document_repository: DocumentRepository):
|
||||
self.repository = document_repository
|
||||
|
||||
async def compute_checksum(self, content: str) -> str:
|
||||
"""Compute SHA-256 checksum of content."""
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
async def scan_files(self, directory: Path) -> dict[str, str]:
|
||||
"""
|
||||
Scan directory for files and their checksums.
|
||||
Only processes files, ignores directories.
|
||||
|
||||
Args:
|
||||
directory: Root directory to scan
|
||||
|
||||
Returns:
|
||||
Dict mapping paths to checksums
|
||||
|
||||
Raises:
|
||||
SyncError: If any file cannot be read
|
||||
"""
|
||||
logger.debug(f"Scanning directory: {directory}")
|
||||
files = {}
|
||||
errors = []
|
||||
|
||||
for path in directory.rglob('*'):
|
||||
if path.is_file():
|
||||
try:
|
||||
content = path.read_text()
|
||||
checksum = await self.compute_checksum(content)
|
||||
# Store path relative to root directory
|
||||
rel_path = str(path.relative_to(directory))
|
||||
files[rel_path] = checksum
|
||||
except Exception as e:
|
||||
errors.append(f"Failed to read {path}: {e}")
|
||||
|
||||
if errors:
|
||||
raise SyncError("Failed to read files:\n" + "\n".join(errors))
|
||||
|
||||
logger.debug(f"Found {len(files)} files")
|
||||
return files
|
||||
|
||||
async def find_changes(self, current_files: dict[str, str]) -> SyncReport:
|
||||
"""
|
||||
Find changes between filesystem and database.
|
||||
|
||||
Args:
|
||||
current_files: Dict mapping paths to checksums
|
||||
|
||||
Returns:
|
||||
SyncReport detailing changes
|
||||
"""
|
||||
logger.debug("Finding changes")
|
||||
|
||||
# Get all documents from DB
|
||||
db_documents = await self.repository.find_all()
|
||||
db_files = {
|
||||
doc.path: doc.checksum
|
||||
for doc in db_documents
|
||||
}
|
||||
|
||||
# Find changes
|
||||
new = set(current_files.keys()) - set(db_files.keys())
|
||||
deleted = set(db_files.keys()) - set(current_files.keys())
|
||||
modified = {
|
||||
path for path in current_files
|
||||
if path in db_files and current_files[path] != db_files[path]
|
||||
}
|
||||
|
||||
return SyncReport(new=new, modified=modified, deleted=deleted)
|
||||
|
||||
async def sync_new_file(self, path: str, directory: Path) -> None:
|
||||
"""
|
||||
Sync a new file.
|
||||
|
||||
Args:
|
||||
path: Relative path to file
|
||||
directory: Root directory
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
full_path = directory / path
|
||||
try:
|
||||
content = full_path.read_text()
|
||||
checksum = await self.compute_checksum(content)
|
||||
await self.repository.create({
|
||||
"path": path,
|
||||
"checksum": checksum
|
||||
})
|
||||
except Exception as e:
|
||||
raise SyncError(f"Failed to sync new file {path}: {e}")
|
||||
|
||||
async def sync_modified_file(self, path: str, directory: Path) -> None:
|
||||
"""
|
||||
Sync a modified file.
|
||||
|
||||
Args:
|
||||
path: Relative path to file
|
||||
directory: Root directory
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
full_path = directory / path
|
||||
try:
|
||||
content = full_path.read_text()
|
||||
checksum = await self.compute_checksum(content)
|
||||
doc = await self.repository.find_by_path(path)
|
||||
if doc:
|
||||
await self.repository.update(doc.id, {"checksum": checksum})
|
||||
else:
|
||||
await self.repository.create({
|
||||
"path": path,
|
||||
"checksum": checksum
|
||||
})
|
||||
except Exception as e:
|
||||
raise SyncError(f"Failed to sync modified file {path}: {e}")
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""
|
||||
Sync filesystem with database.
|
||||
Filesystem is source of truth.
|
||||
|
||||
Args:
|
||||
directory: Root directory to sync
|
||||
|
||||
Returns:
|
||||
SyncReport detailing changes
|
||||
|
||||
Raises:
|
||||
SyncError: If sync fails
|
||||
"""
|
||||
logger.info(f"Starting sync of {directory}")
|
||||
|
||||
# Get current state
|
||||
current_files = await self.scan_files(directory)
|
||||
|
||||
# Find changes
|
||||
changes = await self.find_changes(current_files)
|
||||
logger.info(f"Found changes: {changes}")
|
||||
|
||||
if changes.total_changes == 0:
|
||||
logger.info("No changes detected")
|
||||
return changes
|
||||
|
||||
# Process new files
|
||||
for path in changes.new:
|
||||
logger.debug(f"Processing new file: {path}")
|
||||
await self.sync_new_file(path, directory)
|
||||
|
||||
# Process modified files
|
||||
for path in changes.modified:
|
||||
logger.debug(f"Processing modified file: {path}")
|
||||
await self.sync_modified_file(path, directory)
|
||||
|
||||
# Process deleted files
|
||||
for path in changes.deleted:
|
||||
logger.debug(f"Processing deleted file: {path}")
|
||||
doc = await self.repository.find_by_path(path)
|
||||
if doc:
|
||||
await self.repository.delete(doc.id)
|
||||
|
||||
logger.info("Sync completed successfully")
|
||||
return changes
|
||||
+21
-4
@@ -16,6 +16,7 @@ from basic_memory import db
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.models import Base, Entity as EntityModel
|
||||
from basic_memory.repository.document_repository import DocumentRepository
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.repository.observation_repository import ObservationRepository
|
||||
from basic_memory.repository.relation_repository import RelationRepository
|
||||
@@ -24,6 +25,7 @@ from basic_memory.services import (
|
||||
EntityService,
|
||||
ObservationService,
|
||||
RelationService,
|
||||
DocumentService,
|
||||
)
|
||||
|
||||
|
||||
@@ -68,14 +70,29 @@ async def session_maker(engine_factory) -> async_sessionmaker[AsyncSession]:
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def test_project_path():
|
||||
"""Create a temporary project directory."""
|
||||
"""Create a temporary project directory with standard subdirs."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
project_path = Path(temp_dir) / "test-project"
|
||||
entities_path = project_path / "entities"
|
||||
entities_path.mkdir(parents=True)
|
||||
|
||||
# Create standard directories
|
||||
(project_path / "documents").mkdir(parents=True)
|
||||
(project_path / "entities").mkdir(parents=True)
|
||||
|
||||
yield project_path
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def document_repository(session_maker: async_sessionmaker[AsyncSession]) -> DocumentRepository:
|
||||
"""Create a DocumentRepository instance."""
|
||||
return DocumentRepository(session_maker)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def document_service(document_repository: DocumentRepository) -> DocumentService:
|
||||
"""Create a DocumentService instance."""
|
||||
return DocumentService(document_repository)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def entity_repository(session_maker: async_sessionmaker[AsyncSession]) -> EntityRepository:
|
||||
"""Create an EntityRepository instance."""
|
||||
@@ -132,4 +149,4 @@ async def sample_entity(entity_repository: EntityRepository) -> EntityModel:
|
||||
async def test_entity(entity_service: EntityService) -> EntityModel:
|
||||
"""Create a test entity for reuse in tests."""
|
||||
entity_data = Entity(name="Test Entity", entity_type="test", observations=[])
|
||||
return await entity_service.create_entity(entity_data)
|
||||
return await entity_service.create_entity(entity_data)
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Tests for the DocumentRepository."""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.models import Document
|
||||
from basic_memory.repository.document_repository import DocumentRepository
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def sample_doc(session_maker):
|
||||
"""Create a sample document."""
|
||||
async with db.scoped_session(session_maker) as session:
|
||||
doc = Document(
|
||||
path="test/sample.md",
|
||||
checksum="abc123",
|
||||
doc_metadata={"type": "test", "tags": ["sample"]}
|
||||
)
|
||||
session.add(doc)
|
||||
return doc
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_document_operations(document_repository: DocumentRepository):
|
||||
"""Smoke test for basic document operations."""
|
||||
# Create
|
||||
doc_data = {
|
||||
"path": "test/basic.md",
|
||||
"checksum": "test123",
|
||||
"doc_metadata": {"type": "test"}
|
||||
}
|
||||
doc = await document_repository.create(doc_data)
|
||||
assert doc.path == "test/basic.md"
|
||||
|
||||
# Read
|
||||
found = await document_repository.find_by_path("test/basic.md")
|
||||
assert found is not None
|
||||
assert found.checksum == "test123"
|
||||
|
||||
# Update
|
||||
updated = await document_repository.update(
|
||||
doc.id,
|
||||
{"checksum": "changed123"}
|
||||
)
|
||||
assert updated.checksum == "changed123"
|
||||
|
||||
# Delete
|
||||
result = await document_repository.delete(doc.id)
|
||||
assert result is True
|
||||
|
||||
# Verify deletion
|
||||
not_found = await document_repository.find_by_path("test/basic.md")
|
||||
assert not_found is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_document(document_repository: DocumentRepository):
|
||||
"""Test creating a new document."""
|
||||
doc_data = {
|
||||
"path": "test/doc1.md",
|
||||
"checksum": "xyz789",
|
||||
"doc_metadata": {
|
||||
"type": "note",
|
||||
"tags": ["test"]
|
||||
}
|
||||
}
|
||||
doc = await document_repository.create(doc_data)
|
||||
|
||||
# Verify returned object
|
||||
assert doc.path == "test/doc1.md"
|
||||
assert doc.checksum == "xyz789"
|
||||
assert doc.doc_metadata == {"type": "note", "tags": ["test"]}
|
||||
assert isinstance(doc.created_at, datetime)
|
||||
|
||||
# Verify in database
|
||||
async with db.scoped_session(document_repository.session_maker) as session:
|
||||
stmt = select(Document).where(Document.id == doc.id)
|
||||
result = await session.execute(stmt)
|
||||
db_doc = result.scalar_one()
|
||||
assert db_doc.path == doc.path
|
||||
assert db_doc.checksum == doc.checksum
|
||||
assert db_doc.doc_metadata == doc.doc_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_by_path(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test finding a document by path."""
|
||||
found = await document_repository.find_by_path(sample_doc.path)
|
||||
assert found is not None
|
||||
assert found.id == sample_doc.id
|
||||
assert found.path == sample_doc.path
|
||||
assert found.checksum == sample_doc.checksum
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_by_checksum(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test finding documents by checksum."""
|
||||
found = await document_repository.find_by_checksum(sample_doc.checksum)
|
||||
assert len(found) == 1
|
||||
assert found[0].id == sample_doc.id
|
||||
assert found[0].checksum == sample_doc.checksum
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_changed_documents(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test finding changed documents based on checksums."""
|
||||
checksums = {
|
||||
sample_doc.path: "different_checksum", # Changed
|
||||
"new/doc.md": "new123" # New file
|
||||
}
|
||||
|
||||
changed = await document_repository.find_changed(checksums)
|
||||
assert len(changed) == 1
|
||||
assert changed[0].path == sample_doc.path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_document(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test updating a document."""
|
||||
new_metadata = {"type": "updated", "tags": ["modified"]}
|
||||
updated = await document_repository.update(
|
||||
sample_doc.id,
|
||||
{"checksum": "new456", "doc_metadata": new_metadata}
|
||||
)
|
||||
|
||||
assert updated is not None
|
||||
assert updated.checksum == "new456"
|
||||
assert updated.doc_metadata == new_metadata
|
||||
assert updated.path == sample_doc.path # Path unchanged
|
||||
|
||||
# Verify in database
|
||||
async with db.scoped_session(document_repository.session_maker) as session:
|
||||
stmt = select(Document).where(Document.id == sample_doc.id)
|
||||
result = await session.execute(stmt)
|
||||
db_doc = result.scalar_one()
|
||||
assert db_doc.checksum == "new456"
|
||||
assert db_doc.doc_metadata == new_metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_document(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test deleting a document."""
|
||||
result = await document_repository.delete(sample_doc.id)
|
||||
assert result is True
|
||||
|
||||
# Verify deletion
|
||||
found = await document_repository.find_by_path(sample_doc.path)
|
||||
assert found is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unique_path_constraint(document_repository: DocumentRepository, sample_doc):
|
||||
"""Test that document paths must be unique."""
|
||||
# Try to create a document with same path
|
||||
doc_data = {
|
||||
"path": sample_doc.path, # Same path
|
||||
"checksum": "different",
|
||||
"doc_metadata": {"type": "duplicate"}
|
||||
}
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
await document_repository.create(doc_data)
|
||||
assert "UNIQUE constraint" in str(exc_info.value)
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Tests for DocumentService."""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from pathlib import Path
|
||||
import stat
|
||||
|
||||
from basic_memory.services.document_service import (
|
||||
DocumentService,
|
||||
DocumentNotFoundError,
|
||||
DocumentWriteError
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def document_service(document_repository) -> DocumentService:
|
||||
"""Create DocumentService instance."""
|
||||
return DocumentService(document_repository)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def test_doc_path(tmp_path) -> Path:
|
||||
"""Create a test document directory."""
|
||||
doc_dir = tmp_path / "test_docs"
|
||||
doc_dir.mkdir()
|
||||
return doc_dir / "test.md"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_document_file_first(document_service, test_doc_path):
|
||||
"""Test that files are written before database records."""
|
||||
content = "# Test Document\n\nThis is a test."
|
||||
doc = await document_service.create_document(
|
||||
str(test_doc_path),
|
||||
content,
|
||||
{"type": "test"}
|
||||
)
|
||||
|
||||
# Verify both file and database record
|
||||
assert test_doc_path.exists()
|
||||
assert test_doc_path.read_text() == content
|
||||
assert doc.path == str(test_doc_path)
|
||||
assert doc.doc_metadata == {"type": "test"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_document_unwriteable_directory(document_service, tmp_path):
|
||||
"""Test error when trying to write to an unwriteable directory."""
|
||||
# Create parent directory without write permissions
|
||||
parent_dir = tmp_path / "unwriteable"
|
||||
parent_dir.mkdir()
|
||||
parent_dir.chmod(stat.S_IREAD) # Read-only
|
||||
|
||||
bad_path = parent_dir / "test.md"
|
||||
content = "# Test"
|
||||
|
||||
with pytest.raises(DocumentWriteError):
|
||||
await document_service.create_document(str(bad_path), content)
|
||||
|
||||
# Verify directory is still read-only
|
||||
assert not os.access(parent_dir, os.W_OK)
|
||||
# Verify no database record
|
||||
doc = await document_service.repository.find_by_path(str(bad_path))
|
||||
assert doc is None
|
||||
|
||||
# Clean up - make writable again so it can be deleted
|
||||
parent_dir.chmod(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_nonexistent_file(document_service, test_doc_path):
|
||||
"""Test deleting a file that doesn't exist."""
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
await document_service.delete_document(str(test_doc_path))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_nonexistent_file(document_service, test_doc_path):
|
||||
"""Test updating a file that doesn't exist."""
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
await document_service.update_document(
|
||||
str(test_doc_path),
|
||||
"new content"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_file_exists_no_record(document_service, test_doc_path):
|
||||
"""Test reading a file that exists but has no database record."""
|
||||
content = "# Test\nNo record yet"
|
||||
test_doc_path.write_text(content)
|
||||
|
||||
# Reading should create the record
|
||||
doc, read_content = await document_service.read_document(str(test_doc_path))
|
||||
|
||||
assert read_content == content
|
||||
assert doc is not None
|
||||
assert doc.path == str(test_doc_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_no_file(document_service, test_doc_path):
|
||||
"""Test reading a non-existent file."""
|
||||
with pytest.raises(DocumentNotFoundError):
|
||||
await document_service.read_document(str(test_doc_path))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_file_exists_no_record(document_service, test_doc_path):
|
||||
"""Test updating a file that exists but has no database record."""
|
||||
# Create file without record
|
||||
test_doc_path.write_text("original content")
|
||||
|
||||
# Update should create record
|
||||
new_content = "updated content"
|
||||
doc = await document_service.update_document(
|
||||
str(test_doc_path),
|
||||
new_content,
|
||||
{"status": "updated"}
|
||||
)
|
||||
|
||||
assert doc is not None
|
||||
assert doc.path == str(test_doc_path)
|
||||
assert doc.doc_metadata == {"status": "updated"}
|
||||
assert test_doc_path.read_text() == new_content
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Tests for FileSyncService."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from basic_memory.services.file_sync_service import FileSyncService, SyncError
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def file_sync_service(document_repository) -> FileSyncService:
|
||||
"""Create FileSyncService instance."""
|
||||
return FileSyncService(document_repository)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def docs_dir(test_project_path) -> Path:
|
||||
"""Get documents directory."""
|
||||
return test_project_path / "documents"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def sample_files(docs_dir) -> dict[str, str]:
|
||||
"""Create some sample test files."""
|
||||
# Create test structure
|
||||
design_dir = docs_dir / "design"
|
||||
notes_dir = docs_dir / "notes"
|
||||
design_dir.mkdir(exist_ok=True)
|
||||
notes_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Map of relative paths to content
|
||||
files = {
|
||||
"design/architecture.md": "# Architecture\nSome design notes",
|
||||
"notes/meeting.md": "# Meeting Notes\nDiscussion points",
|
||||
"README.md": "# Project\nOverview doc",
|
||||
}
|
||||
|
||||
# Create files with full paths
|
||||
for rel_path, content in files.items():
|
||||
full_path = docs_dir / rel_path
|
||||
full_path.write_text(content)
|
||||
|
||||
return files
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scan_files(file_sync_service, docs_dir, sample_files):
|
||||
"""Test scanning directory for files."""
|
||||
scanned = await file_sync_service.scan_files(docs_dir)
|
||||
|
||||
# Should find all files
|
||||
assert len(scanned) == len(sample_files)
|
||||
|
||||
# Paths should be relative and match sample files
|
||||
assert set(scanned.keys()) == set(sample_files.keys())
|
||||
|
||||
# All files should have checksums
|
||||
assert all(isinstance(checksum, str) for checksum in scanned.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_new_files(file_sync_service, docs_dir, sample_files):
|
||||
"""Test detecting new files."""
|
||||
changes = await file_sync_service.find_changes(
|
||||
await file_sync_service.scan_files(docs_dir)
|
||||
)
|
||||
|
||||
# All files should be new
|
||||
assert len(changes.new) == len(sample_files)
|
||||
assert len(changes.modified) == 0
|
||||
assert len(changes.deleted) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_modified_files(file_sync_service, docs_dir, sample_files):
|
||||
"""Test detecting modified files."""
|
||||
# First sync to create DB records
|
||||
await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Modify a file
|
||||
mod_path = docs_dir / "design/architecture.md"
|
||||
mod_path.write_text("# Updated Architecture")
|
||||
|
||||
# Check changes
|
||||
changes = await file_sync_service.find_changes(
|
||||
await file_sync_service.scan_files(docs_dir)
|
||||
)
|
||||
|
||||
assert len(changes.modified) == 1
|
||||
assert "design/architecture.md" in changes.modified
|
||||
assert len(changes.new) == 0
|
||||
assert len(changes.deleted) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_deleted_files(file_sync_service, docs_dir, sample_files):
|
||||
"""Test detecting deleted files."""
|
||||
# First sync to create DB records
|
||||
await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Delete a file
|
||||
del_path = docs_dir / "notes/meeting.md"
|
||||
del_path.unlink()
|
||||
|
||||
# Check changes
|
||||
changes = await file_sync_service.find_changes(
|
||||
await file_sync_service.scan_files(docs_dir)
|
||||
)
|
||||
|
||||
assert len(changes.deleted) == 1
|
||||
assert "notes/meeting.md" in changes.deleted
|
||||
assert len(changes.new) == 0
|
||||
assert len(changes.modified) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_sync_process(file_sync_service, docs_dir, sample_files):
|
||||
"""Test full sync process with various changes."""
|
||||
# First sync to create initial state
|
||||
initial_sync = await file_sync_service.sync(docs_dir)
|
||||
assert initial_sync.total_changes == len(sample_files)
|
||||
|
||||
# Make some changes:
|
||||
# 1. Add new file
|
||||
(docs_dir / "notes").mkdir(exist_ok=True) # Ensure parent exists
|
||||
(docs_dir / "notes/todo.md").write_text("# TODO\n- First item")
|
||||
|
||||
# 2. Modify existing file
|
||||
(docs_dir / "README.md").write_text("# Updated Project")
|
||||
|
||||
# 3. Delete a file
|
||||
(docs_dir / "design/architecture.md").unlink()
|
||||
|
||||
# Run sync
|
||||
changes = await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Verify changes
|
||||
assert len(changes.new) == 1
|
||||
assert "notes/todo.md" in changes.new
|
||||
|
||||
assert len(changes.modified) == 1
|
||||
assert "README.md" in changes.modified
|
||||
|
||||
assert len(changes.deleted) == 1
|
||||
assert "design/architecture.md" in changes.deleted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_changes_sync(file_sync_service, docs_dir, sample_files):
|
||||
"""Test sync when no changes are present."""
|
||||
# First sync to create initial state
|
||||
await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Sync again immediately
|
||||
changes = await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Should detect no changes
|
||||
assert changes.total_changes == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_empty_directory(file_sync_service, docs_dir):
|
||||
"""Test syncing an empty directory."""
|
||||
changes = await file_sync_service.sync(docs_dir)
|
||||
assert changes.total_changes == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_on_unreadable_file(file_sync_service, docs_dir):
|
||||
"""Test handling of unreadable files during sync."""
|
||||
# Create file without read permissions
|
||||
bad_file = docs_dir / "bad.md"
|
||||
bad_file.write_text("test")
|
||||
bad_file.chmod(0o000) # Remove all permissions
|
||||
|
||||
with pytest.raises(SyncError):
|
||||
await file_sync_service.sync(docs_dir)
|
||||
|
||||
# Clean up
|
||||
bad_file.chmod(0o666) # Make readable/writable for cleanup
|
||||
@@ -186,10 +186,12 @@ dependencies = [
|
||||
{ name = "greenlet" },
|
||||
{ name = "icecream" },
|
||||
{ name = "loguru" },
|
||||
{ name = "markdown-it-py" },
|
||||
{ name = "mcp" },
|
||||
{ name = "pydantic", extra = ["email", "timezone"] },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "pyright" },
|
||||
{ name = "python-frontmatter" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "rich" },
|
||||
{ name = "sqlalchemy" },
|
||||
@@ -217,6 +219,7 @@ requires-dist = [
|
||||
{ name = "greenlet", specifier = ">=3.1.1" },
|
||||
{ name = "icecream", specifier = ">=2.1.3" },
|
||||
{ name = "loguru", specifier = ">=0.7.3" },
|
||||
{ name = "markdown-it-py", specifier = ">=3.0.0" },
|
||||
{ name = "mcp", specifier = ">=1.1.0" },
|
||||
{ name = "pydantic", extras = ["email", "timezone"], specifier = ">=2.10.3" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.6.1" },
|
||||
@@ -225,6 +228,7 @@ requires-dist = [
|
||||
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },
|
||||
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" },
|
||||
{ name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.12.0" },
|
||||
{ name = "python-frontmatter", specifier = ">=1.1.0" },
|
||||
{ name = "pyyaml", specifier = ">=6.0.1" },
|
||||
{ name = "rich", specifier = ">=13.7.0" },
|
||||
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.6" },
|
||||
@@ -1295,6 +1299,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-frontmatter"
|
||||
version = "1.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pyyaml" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/de/910fa208120314a12f9a88ea63e03707261692af782c99283f1a2c8a5e6f/python-frontmatter-1.1.0.tar.gz", hash = "sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d", size = 16256 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/49/87/3c8da047b3ec5f99511d1b4d7a5bc72d4b98751c7e78492d14dc736319c5/python_frontmatter-1.1.0-py3-none-any.whl", hash = "sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1", size = 9834 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-multipart"
|
||||
version = "0.0.19"
|
||||
|
||||
Reference in New Issue
Block a user