mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
chore: formatting
This commit is contained in:
@@ -42,4 +42,13 @@ installer-win:
|
||||
update-deps:
|
||||
uv lock f--upgrade
|
||||
|
||||
check: lint format type-check test
|
||||
check: lint format type-check test
|
||||
|
||||
|
||||
# Target for generating Alembic migrations with a message from command line
|
||||
migration:
|
||||
@if [ -z "$(m)" ]; then \
|
||||
echo "Usage: make migration m=\"Your migration message\""; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
cd src/basic_memory/alembic && alembic revision --autogenerate -m "$(m)"
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Alembic environment configuration."""
|
||||
|
||||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
@@ -8,6 +9,10 @@ from sqlalchemy import pool
|
||||
from alembic import context
|
||||
|
||||
from basic_memory.models import Base
|
||||
|
||||
# set config.env to "test" for pytest to prevent logging to file in utils.setup_logging()
|
||||
os.environ["BASIC_MEMORY_ENV"] = "test"
|
||||
|
||||
from basic_memory.config import config as app_config
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
@@ -18,7 +23,7 @@ config = context.config
|
||||
sqlalchemy_url = f"sqlite:///{app_config.database_path}"
|
||||
config.set_main_option("sqlalchemy.url", sqlalchemy_url)
|
||||
|
||||
#print(f"Using SQLAlchemy URL: {sqlalchemy_url}")
|
||||
# print(f"Using SQLAlchemy URL: {sqlalchemy_url}")
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
if config.config_file_name is not None:
|
||||
@@ -29,6 +34,14 @@ if config.config_file_name is not None:
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
# Add this function to tell Alembic what to include/exclude
|
||||
def include_object(object, name, type_, reflected, compare_to):
|
||||
# Ignore SQLite FTS tables
|
||||
if type_ == "table" and name.startswith("search_index"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
@@ -46,6 +59,8 @@ def run_migrations_offline() -> None:
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
include_object=include_object,
|
||||
render_as_batch=True,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
@@ -65,7 +80,12 @@ def run_migrations_online() -> None:
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
include_object=include_object,
|
||||
render_as_batch=True,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""remove required from entity.permalink
|
||||
|
||||
Revision ID: 502b60eaa905
|
||||
Revises: b3c3938bacdb
|
||||
Create Date: 2025-02-24 13:33:09.790951
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "502b60eaa905"
|
||||
down_revision: Union[str, None] = "b3c3938bacdb"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
with op.batch_alter_table("entity", schema=None) as batch_op:
|
||||
batch_op.alter_column("permalink", existing_type=sa.VARCHAR(), nullable=True)
|
||||
batch_op.drop_index("ix_entity_permalink")
|
||||
batch_op.create_index(batch_op.f("ix_entity_permalink"), ["permalink"], unique=False)
|
||||
batch_op.drop_constraint("uix_entity_permalink", type_="unique")
|
||||
batch_op.create_index(
|
||||
"uix_entity_permalink",
|
||||
["permalink"],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
|
||||
)
|
||||
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
with op.batch_alter_table("entity", schema=None) as batch_op:
|
||||
batch_op.drop_index(
|
||||
"uix_entity_permalink",
|
||||
sqlite_where=sa.text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
|
||||
)
|
||||
batch_op.create_unique_constraint("uix_entity_permalink", ["permalink"])
|
||||
batch_op.drop_index(batch_op.f("ix_entity_permalink"))
|
||||
batch_op.create_index("ix_entity_permalink", ["permalink"], unique=1)
|
||||
batch_op.alter_column("permalink", existing_type=sa.VARCHAR(), nullable=False)
|
||||
|
||||
# ### end Alembic commands ###
|
||||
@@ -5,16 +5,15 @@ Revises: 3dae7c7b1564
|
||||
Create Date: 2025-02-22 14:59:30.668466
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from alembic.context import get_context
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = 'b3c3938bacdb'
|
||||
down_revision: Union[str, None] = '3dae7c7b1564'
|
||||
revision: str = "b3c3938bacdb"
|
||||
down_revision: Union[str, None] = "3dae7c7b1564"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
@@ -22,29 +21,24 @@ depends_on: Union[str, Sequence[str], None] = None
|
||||
def upgrade() -> None:
|
||||
# SQLite doesn't support constraint changes through ALTER
|
||||
# Need to recreate table with desired constraints
|
||||
with op.batch_alter_table('relation') as batch_op:
|
||||
with op.batch_alter_table("relation") as batch_op:
|
||||
# Drop existing unique constraint
|
||||
batch_op.drop_constraint('uix_relation', type_='unique')
|
||||
|
||||
batch_op.drop_constraint("uix_relation", type_="unique")
|
||||
|
||||
# Add new constraints
|
||||
batch_op.create_unique_constraint(
|
||||
'uix_relation_from_id_to_id',
|
||||
['from_id', 'to_id', 'relation_type']
|
||||
"uix_relation_from_id_to_id", ["from_id", "to_id", "relation_type"]
|
||||
)
|
||||
batch_op.create_unique_constraint(
|
||||
'uix_relation_from_id_to_name',
|
||||
['from_id', 'to_name', 'relation_type']
|
||||
"uix_relation_from_id_to_name", ["from_id", "to_name", "relation_type"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table('relation') as batch_op:
|
||||
with op.batch_alter_table("relation") as batch_op:
|
||||
# Drop new constraints
|
||||
batch_op.drop_constraint('uix_relation_from_id_to_name', type_='unique')
|
||||
batch_op.drop_constraint('uix_relation_from_id_to_id', type_='unique')
|
||||
|
||||
batch_op.drop_constraint("uix_relation_from_id_to_name", type_="unique")
|
||||
batch_op.drop_constraint("uix_relation_from_id_to_id", type_="unique")
|
||||
|
||||
# Restore original constraint
|
||||
batch_op.create_unique_constraint(
|
||||
'uix_relation',
|
||||
['from_id', 'to_id', 'relation_type']
|
||||
)
|
||||
batch_op.create_unique_constraint("uix_relation", ["from_id", "to_id", "relation_type"])
|
||||
|
||||
@@ -133,7 +133,7 @@ async def delete_entity(
|
||||
return DeleteEntitiesResponse(deleted=False)
|
||||
|
||||
# Delete the entity
|
||||
deleted = await entity_service.delete_entity(entity.permalink)
|
||||
deleted = await entity_service.delete_entity(entity.permalink or entity.id)
|
||||
|
||||
# Remove from search index
|
||||
background_tasks.add_task(search_service.delete_by_permalink, entity.permalink)
|
||||
|
||||
@@ -41,17 +41,19 @@ async def to_graph_context(context, entity_repository: EntityRepository, page: i
|
||||
case SearchItemType.OBSERVATION:
|
||||
assert item.category is not None
|
||||
assert item.content is not None
|
||||
assert item.permalink is not None
|
||||
|
||||
return ObservationSummary(
|
||||
category=item.category, content=item.content, permalink=item.permalink
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
assert item.from_id is not None
|
||||
assert item.permalink is not None
|
||||
from_entity = await entity_repository.find_by_id(item.from_id)
|
||||
assert from_entity is not None
|
||||
assert from_entity.permalink is not None
|
||||
|
||||
to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
|
||||
|
||||
return RelationSummary(
|
||||
permalink=item.permalink,
|
||||
relation_type=item.type,
|
||||
@@ -104,9 +106,11 @@ async def recent(
|
||||
context = await context_service.build_context(
|
||||
types=types, depth=depth, since=since, limit=limit, offset=offset, max_related=max_related
|
||||
)
|
||||
return await to_graph_context(
|
||||
recent_context = await to_graph_context(
|
||||
context, entity_repository=entity_repository, page=page, page_size=page_size
|
||||
)
|
||||
logger.debug(f"Recent context: {recent_context.model_dump_json()}")
|
||||
return recent_context
|
||||
|
||||
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
|
||||
@@ -17,4 +17,4 @@ def mcp(): # pragma: no cover
|
||||
home_dir = config.home
|
||||
logger.info(f"Starting Basic Memory MCP server {basic_memory.__version__}")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
mcp_server.run()
|
||||
mcp_server.run()
|
||||
|
||||
@@ -141,4 +141,4 @@ def status(
|
||||
except Exception as e:
|
||||
logger.exception(f"Error checking status: {e}")
|
||||
typer.echo(f"Error checking status: {e}", err=True)
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
|
||||
@@ -160,8 +160,10 @@ async def run_sync(verbose: bool = False, watch: bool = False, console_status: b
|
||||
file_service=sync_service.entity_service.file_service,
|
||||
config=config,
|
||||
)
|
||||
await watch_service.handle_changes(config.home)
|
||||
await watch_service.run(console_status=console_status) # pragma: no cover
|
||||
# full sync
|
||||
await sync_service.sync(config.home)
|
||||
# watch changes
|
||||
await watch_service.run() # pragma: no cover
|
||||
else:
|
||||
# one time sync
|
||||
knowledge_changes = await sync_service.sync(config.home)
|
||||
@@ -186,18 +188,15 @@ def sync(
|
||||
"-w",
|
||||
help="Start watching for changes after sync.",
|
||||
),
|
||||
console_status: bool = typer.Option(
|
||||
False, "--console-status", "-c", help="Show live console status"
|
||||
),
|
||||
) -> None:
|
||||
"""Sync knowledge files with the database."""
|
||||
try:
|
||||
# Run sync
|
||||
asyncio.run(run_sync(verbose=verbose, watch=watch, console_status=console_status))
|
||||
asyncio.run(run_sync(verbose=verbose, watch=watch))
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
if not isinstance(e, typer.Exit):
|
||||
logger.exception("Sync failed")
|
||||
typer.echo(f"Error during sync: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
raise
|
||||
|
||||
@@ -9,7 +9,6 @@ from rich import print as rprint
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.mcp.tools import build_context as mcp_build_context
|
||||
from basic_memory.mcp.tools import get_entity as mcp_get_entity
|
||||
from basic_memory.mcp.tools import read_resource as mcp_read_resource
|
||||
from basic_memory.mcp.tools import read_note as mcp_read_note
|
||||
from basic_memory.mcp.tools import recent_activity as mcp_recent_activity
|
||||
from basic_memory.mcp.tools import search as mcp_search
|
||||
@@ -156,14 +155,3 @@ def get_entity(identifier: str):
|
||||
typer.echo(f"Error during get_entity: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
|
||||
@tool_app.command()
|
||||
def read_resource(identifier: str):
|
||||
try:
|
||||
entity = asyncio.run(read_resource(identifier=identifier))
|
||||
rprint(entity.model_dump_json(indent=2))
|
||||
except Exception as e: # pragma: no cover
|
||||
if not isinstance(e, typer.Exit):
|
||||
typer.echo(f"Error during get_entity: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise
|
||||
|
||||
@@ -35,7 +35,7 @@ class ProjectConfig(BaseSettings):
|
||||
default=500, description="Milliseconds to wait after changes before syncing", gt=0
|
||||
)
|
||||
|
||||
log_level: str = "DEBUG"
|
||||
log_level: str = "DEBUG"
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="BASIC_MEMORY_",
|
||||
|
||||
@@ -47,6 +47,7 @@ async def compute_checksum(content: Union[str, bytes]) -> str:
|
||||
logger.error(f"Failed to compute checksum: {e}")
|
||||
raise FileError(f"Failed to compute checksum: {e}")
|
||||
|
||||
|
||||
async def ensure_directory(path: Path) -> None:
|
||||
"""
|
||||
Ensure directory exists, creating if necessary.
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""MCP server for basic-memory."""
|
||||
"""MCP server for basic-memory."""
|
||||
|
||||
@@ -3,18 +3,18 @@
|
||||
Creates and configures the shared MCP instance and handles server startup.
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
from loguru import logger # pragma: no cover
|
||||
|
||||
from basic_memory.config import config
|
||||
from basic_memory.config import config # pragma: no cover
|
||||
|
||||
# Import shared mcp instance
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.server import mcp # pragma: no cover
|
||||
|
||||
# Import tools to register them
|
||||
import basic_memory.mcp.tools # noqa: F401
|
||||
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
home_dir = config.home
|
||||
logger.info("Starting Basic Memory MCP server")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
|
||||
@@ -4,6 +4,7 @@ This package provides the complete set of tools for interacting with
|
||||
Basic Memory through the MCP protocol. Importing this module registers
|
||||
all tools with the MCP server.
|
||||
"""
|
||||
|
||||
# Import tools to register them with MCP
|
||||
from basic_memory.mcp.tools.resource import read_resource
|
||||
from basic_memory.mcp.tools.memory import build_context, recent_activity
|
||||
@@ -30,4 +31,5 @@ __all__ = [
|
||||
"read_note",
|
||||
"write_note",
|
||||
# files
|
||||
"read_resource",
|
||||
]
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from loguru import logger
|
||||
import logfire
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.async_client import client
|
||||
@@ -10,16 +9,19 @@ import base64
|
||||
import io
|
||||
from PIL import Image as PILImage
|
||||
|
||||
|
||||
def calculate_target_params(content_length):
|
||||
"""Calculate initial quality and size based on input file size"""
|
||||
target_size = 350000 # Reduced target for more safety margin
|
||||
ratio = content_length / target_size
|
||||
|
||||
logger.debug("Calculating target parameters",
|
||||
content_length=content_length,
|
||||
ratio=ratio,
|
||||
target_size=target_size)
|
||||
|
||||
|
||||
logger.debug(
|
||||
"Calculating target parameters",
|
||||
content_length=content_length,
|
||||
ratio=ratio,
|
||||
target_size=target_size,
|
||||
)
|
||||
|
||||
if ratio > 4:
|
||||
# Very large images - start very aggressive
|
||||
return 50, 600 # Lower initial quality and size
|
||||
@@ -28,117 +30,123 @@ def calculate_target_params(content_length):
|
||||
else:
|
||||
return 70, 1000
|
||||
|
||||
|
||||
def resize_image(img, max_size):
|
||||
"""Resize image maintaining aspect ratio"""
|
||||
original_dimensions = {"width": img.width, "height": img.height}
|
||||
|
||||
|
||||
if img.width > max_size or img.height > max_size:
|
||||
ratio = min(max_size / img.width, max_size / img.height)
|
||||
new_size = (int(img.width * ratio), int(img.height * ratio))
|
||||
logger.debug("Resizing image",
|
||||
original=original_dimensions,
|
||||
target=new_size,
|
||||
ratio=ratio)
|
||||
logger.debug("Resizing image", original=original_dimensions, target=new_size, ratio=ratio)
|
||||
return img.resize(new_size, PILImage.Resampling.LANCZOS)
|
||||
|
||||
|
||||
logger.debug("No resize needed", dimensions=original_dimensions)
|
||||
return img
|
||||
|
||||
|
||||
def optimize_image(img, content_length, max_output_bytes=350000):
|
||||
"""Iteratively optimize image with aggressive size reduction"""
|
||||
stats = {
|
||||
"dimensions": {"width": img.width, "height": img.height},
|
||||
"mode": img.mode,
|
||||
"estimated_memory": (img.width * img.height * len(img.getbands()))
|
||||
"estimated_memory": (img.width * img.height * len(img.getbands())),
|
||||
}
|
||||
|
||||
|
||||
initial_quality, initial_size = calculate_target_params(content_length)
|
||||
|
||||
logger.debug("Starting optimization",
|
||||
image_stats=stats,
|
||||
content_length=content_length,
|
||||
initial_quality=initial_quality,
|
||||
initial_size=initial_size,
|
||||
max_output_bytes=max_output_bytes)
|
||||
|
||||
|
||||
logger.debug(
|
||||
"Starting optimization",
|
||||
image_stats=stats,
|
||||
content_length=content_length,
|
||||
initial_quality=initial_quality,
|
||||
initial_size=initial_size,
|
||||
max_output_bytes=max_output_bytes,
|
||||
)
|
||||
|
||||
quality = initial_quality
|
||||
size = initial_size
|
||||
|
||||
|
||||
# Convert to RGB if needed
|
||||
if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
|
||||
img = img.convert('RGB')
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
img = img.convert("RGB")
|
||||
logger.debug("Converted to RGB mode")
|
||||
|
||||
|
||||
iteration = 0
|
||||
min_size = 300 # Absolute minimum size
|
||||
min_quality = 20 # Absolute minimum quality
|
||||
|
||||
|
||||
while True:
|
||||
iteration += 1
|
||||
buf = io.BytesIO()
|
||||
resized = resize_image(img, size)
|
||||
|
||||
resized.save(buf, format='JPEG',
|
||||
quality=quality,
|
||||
optimize=True,
|
||||
progressive=True,
|
||||
subsampling='4:2:0')
|
||||
|
||||
|
||||
resized.save(
|
||||
buf,
|
||||
format="JPEG",
|
||||
quality=quality,
|
||||
optimize=True,
|
||||
progressive=True,
|
||||
subsampling="4:2:0",
|
||||
)
|
||||
|
||||
output_size = buf.getbuffer().nbytes
|
||||
reduction_ratio = output_size / content_length
|
||||
|
||||
logger.debug("Optimization attempt",
|
||||
iteration=iteration,
|
||||
quality=quality,
|
||||
size=size,
|
||||
output_bytes=output_size,
|
||||
target_bytes=max_output_bytes,
|
||||
reduction_ratio=f"{reduction_ratio:.2f}")
|
||||
|
||||
|
||||
logger.debug(
|
||||
"Optimization attempt",
|
||||
iteration=iteration,
|
||||
quality=quality,
|
||||
size=size,
|
||||
output_bytes=output_size,
|
||||
target_bytes=max_output_bytes,
|
||||
reduction_ratio=f"{reduction_ratio:.2f}",
|
||||
)
|
||||
|
||||
if output_size < max_output_bytes:
|
||||
logger.info("Image optimization complete",
|
||||
final_size=output_size,
|
||||
quality=quality,
|
||||
dimensions={"width": resized.width, "height": resized.height},
|
||||
reduction_ratio=f"{reduction_ratio:.2f}")
|
||||
logger.info(
|
||||
"Image optimization complete",
|
||||
final_size=output_size,
|
||||
quality=quality,
|
||||
dimensions={"width": resized.width, "height": resized.height},
|
||||
reduction_ratio=f"{reduction_ratio:.2f}",
|
||||
)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# Very aggressive reduction for large files
|
||||
if content_length > 2000000: # 2MB+
|
||||
if content_length > 2000000: # 2MB+ # pragma: no cover
|
||||
quality = max(min_quality, quality - 20)
|
||||
size = max(min_size, int(size * 0.6))
|
||||
elif content_length > 1000000: # 1MB+
|
||||
elif content_length > 1000000: # 1MB+ # pragma: no cover
|
||||
quality = max(min_quality, quality - 15)
|
||||
size = max(min_size, int(size * 0.7))
|
||||
else:
|
||||
quality = max(min_quality, quality - 10)
|
||||
size = max(min_size, int(size * 0.8))
|
||||
|
||||
logger.debug("Reducing parameters",
|
||||
new_quality=quality,
|
||||
new_size=size)
|
||||
|
||||
quality = max(min_quality, quality - 10) # pragma: no cover
|
||||
size = max(min_size, int(size * 0.8)) # pragma: no cover
|
||||
|
||||
logger.debug("Reducing parameters", new_quality=quality, new_size=size) # pragma: no cover
|
||||
|
||||
# If we've hit minimum values and still too big
|
||||
if quality <= min_quality and size <= min_size:
|
||||
logger.warning("Reached minimum parameters",
|
||||
final_size=output_size,
|
||||
over_limit_by=output_size - max_output_bytes)
|
||||
if quality <= min_quality and size <= min_size: # pragma: no cover
|
||||
logger.warning(
|
||||
"Reached minimum parameters",
|
||||
final_size=output_size,
|
||||
over_limit_by=output_size - max_output_bytes,
|
||||
)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
@mcp.tool(description="Read a single file's content by path or permalink")
|
||||
async def read_resource(path: str) -> dict:
|
||||
"""Get a file's raw content."""
|
||||
logger.info("Reading resource", path=path)
|
||||
|
||||
|
||||
url = memory_url_path(path)
|
||||
response = await call_get(client, f"/resource/{url}")
|
||||
content_type = response.headers.get("content-type", "application/octet-stream")
|
||||
content_length = int(response.headers.get("content-length", 0))
|
||||
|
||||
logger.debug("Resource metadata",
|
||||
content_type=content_type,
|
||||
size=content_length,
|
||||
path=path)
|
||||
|
||||
logger.debug("Resource metadata", content_type=content_type, size=content_length, path=path)
|
||||
|
||||
# Handle text or json
|
||||
if content_type.startswith("text/") or content_type == "application/json":
|
||||
@@ -149,31 +157,30 @@ async def read_resource(path: str) -> dict:
|
||||
"content_type": content_type,
|
||||
"encoding": "utf-8",
|
||||
}
|
||||
|
||||
|
||||
# Handle images
|
||||
elif content_type.startswith("image/"):
|
||||
logger.debug("Processing image")
|
||||
img = PILImage.open(io.BytesIO(response.content))
|
||||
img_bytes = optimize_image(img, content_length)
|
||||
|
||||
|
||||
return {
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": base64.b64encode(img_bytes).decode("utf-8")
|
||||
}
|
||||
"data": base64.b64encode(img_bytes).decode("utf-8"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Handle other file types
|
||||
else:
|
||||
logger.debug("Processing binary resource")
|
||||
logger.debug(f"Processing binary resource content_type {content_type}")
|
||||
if content_length > 350000:
|
||||
logger.warning("Document too large for response",
|
||||
size=content_length)
|
||||
logger.warning("Document too large for response", size=content_length)
|
||||
return {
|
||||
"type": "error",
|
||||
"error": f"Document size {content_length} bytes exceeds maximum allowed size"
|
||||
"error": f"Document size {content_length} bytes exceeds maximum allowed size",
|
||||
}
|
||||
return {
|
||||
"type": "document",
|
||||
|
||||
@@ -12,6 +12,7 @@ from sqlalchemy import (
|
||||
DateTime,
|
||||
Index,
|
||||
JSON,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
@@ -32,11 +33,18 @@ class Entity(Base):
|
||||
|
||||
__tablename__ = "entity"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("permalink", name="uix_entity_permalink"), # Make permalink unique
|
||||
# Regular indexes
|
||||
Index("ix_entity_type", "entity_type"),
|
||||
Index("ix_entity_title", "title"),
|
||||
Index("ix_entity_created_at", "created_at"), # For timeline queries
|
||||
Index("ix_entity_updated_at", "updated_at"), # For timeline queries
|
||||
# Unique index only for markdown files with non-null permalinks
|
||||
Index(
|
||||
"uix_entity_permalink",
|
||||
"permalink",
|
||||
unique=True,
|
||||
sqlite_where=text("content_type = 'text/markdown' AND permalink IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
# Core identity
|
||||
@@ -46,8 +54,8 @@ class Entity(Base):
|
||||
entity_metadata: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True)
|
||||
content_type: Mapped[str] = mapped_column(String)
|
||||
|
||||
# Normalized path for URIs
|
||||
permalink: Mapped[str] = mapped_column(String, unique=True, index=True)
|
||||
# Normalized path for URIs - required for markdown files only
|
||||
permalink: Mapped[Optional[str]] = mapped_column(String, nullable=True, index=True)
|
||||
# Actual filesystem relative path
|
||||
file_path: Mapped[str] = mapped_column(String, unique=True, index=True)
|
||||
# checksum of file
|
||||
@@ -133,7 +141,9 @@ class Relation(Base):
|
||||
__tablename__ = "relation"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation_from_id_to_id"),
|
||||
UniqueConstraint("from_id", "to_name", "relation_type", name="uix_relation_from_id_to_name"),
|
||||
UniqueConstraint(
|
||||
"from_id", "to_name", "relation_type", name="uix_relation_from_id_to_name"
|
||||
),
|
||||
Index("ix_relation_type", "relation_type"),
|
||||
Index("ix_relation_from_id", "from_id"), # Add FK indexes
|
||||
Index("ix_relation_to_id", "to_id"),
|
||||
@@ -161,13 +171,13 @@ class Relation(Base):
|
||||
Format: source/relation_type/target
|
||||
Example: "specs/search/implements/features/search-ui"
|
||||
"""
|
||||
# Only create permalinks when both source and target have permalinks
|
||||
from_permalink = self.from_entity.permalink or self.from_entity.file_path
|
||||
|
||||
if self.to_entity:
|
||||
return generate_permalink(
|
||||
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
|
||||
)
|
||||
return generate_permalink(
|
||||
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
|
||||
)
|
||||
to_permalink = self.to_entity.permalink or self.to_entity.file_path
|
||||
return generate_permalink(f"{from_permalink}/{self.relation_type}/{to_permalink}")
|
||||
return generate_permalink(f"{from_permalink}/{self.relation_type}/{self.to_name}")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')"
|
||||
return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, to_name={self.to_name}, type='{self.relation_type}')" # pragma: no cover
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""Repository for managing entities in the knowledge graph."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Sequence, Union, Dict
|
||||
from typing import List, Optional, Sequence, Union
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.orm.interfaces import LoaderOption
|
||||
|
||||
@@ -263,4 +263,4 @@ class Repository[T: Base]:
|
||||
def get_load_options(self) -> List[LoaderOption]:
|
||||
"""Get list of loader options for eager loading relationships.
|
||||
Override in subclasses to specify what to load."""
|
||||
return []
|
||||
return []
|
||||
|
||||
@@ -21,8 +21,8 @@ class SearchIndexRow:
|
||||
|
||||
id: int
|
||||
type: str
|
||||
permalink: str
|
||||
file_path: str
|
||||
permalink: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
|
||||
# date values
|
||||
|
||||
@@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
|
||||
def normalize_memory_url(url: str) -> str:
|
||||
def normalize_memory_url(url: str | None) -> str:
|
||||
"""Normalize a MemoryUrl string.
|
||||
|
||||
Args:
|
||||
@@ -24,6 +24,9 @@ def normalize_memory_url(url: str) -> str:
|
||||
>>> normalize_memory_url("memory://specs/search")
|
||||
'memory://specs/search'
|
||||
"""
|
||||
if not url:
|
||||
return ""
|
||||
|
||||
clean_path = url.removeprefix("memory://")
|
||||
return f"memory://{clean_path}"
|
||||
|
||||
@@ -59,7 +62,7 @@ class EntitySummary(BaseModel):
|
||||
"""Simplified entity representation."""
|
||||
|
||||
type: str = "entity"
|
||||
permalink: str
|
||||
permalink: Optional[str]
|
||||
title: str
|
||||
file_path: str
|
||||
created_at: datetime
|
||||
|
||||
@@ -127,7 +127,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
await self.create_entity_from_markdown(file_path, entity_markdown)
|
||||
|
||||
# add relations
|
||||
entity = await self.update_entity_relations(file_path, entity_markdown)
|
||||
entity = await self.update_entity_relations(str(file_path), entity_markdown)
|
||||
|
||||
# Set final checksum to mark complete
|
||||
return await self.repository.update(entity.id, {"checksum": checksum})
|
||||
@@ -152,20 +152,25 @@ class EntityService(BaseService[EntityModel]):
|
||||
entity = await self.update_entity_and_observations(file_path, entity_markdown)
|
||||
|
||||
# add relations
|
||||
await self.update_entity_relations(file_path, entity_markdown)
|
||||
await self.update_entity_relations(str(file_path), entity_markdown)
|
||||
|
||||
# Set final checksum to match file
|
||||
entity = await self.repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
return entity
|
||||
|
||||
async def delete_entity(self, permalink: str) -> bool:
|
||||
async def delete_entity(self, permalink_or_id: str | int) -> bool:
|
||||
"""Delete entity and its file."""
|
||||
logger.debug(f"Deleting entity: {permalink}")
|
||||
logger.debug(f"Deleting entity: {permalink_or_id}")
|
||||
|
||||
try:
|
||||
# Get entity first for file deletion
|
||||
entity = await self.get_by_permalink(permalink)
|
||||
if isinstance(permalink_or_id, str):
|
||||
entity = await self.get_by_permalink(permalink_or_id)
|
||||
else:
|
||||
entities = await self.get_entities_by_id([permalink_or_id])
|
||||
assert len(entities) == 1, f"Expected 1 entity, got {len(entities)}"
|
||||
entity = entities[0]
|
||||
|
||||
# Delete file first
|
||||
await self.file_service.delete_entity_file(entity)
|
||||
@@ -174,7 +179,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
return await self.repository.delete(entity.id)
|
||||
|
||||
except EntityNotFoundError:
|
||||
logger.info(f"Entity not found: {permalink}")
|
||||
logger.info(f"Entity not found: {permalink_or_id}")
|
||||
return True # Already deleted
|
||||
|
||||
async def get_by_permalink(self, permalink: str) -> EntityModel:
|
||||
|
||||
@@ -157,7 +157,7 @@ class FileService:
|
||||
full_path = path if path.is_absolute() else self.base_path / path
|
||||
|
||||
try:
|
||||
content = path.read_text()
|
||||
content = full_path.read_text()
|
||||
checksum = await file_utils.compute_checksum(content)
|
||||
logger.debug(f"read file: {full_path}, checksum: {checksum}")
|
||||
return content, checksum
|
||||
@@ -201,7 +201,7 @@ class FileService:
|
||||
content = full_path.read_bytes()
|
||||
return await file_utils.compute_checksum(content)
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to compute checksum for {path}: {e}")
|
||||
raise FileError(f"Failed to compute checksum for {path}: {e}")
|
||||
|
||||
@@ -229,7 +229,7 @@ class FileService:
|
||||
content_type = mime_type or "text/plain"
|
||||
return content_type
|
||||
|
||||
def is_markdown(self, path: Union[Path, str]) -> stat_result:
|
||||
def is_markdown(self, path: Union[Path, str]) -> bool:
|
||||
"""
|
||||
Return content_type for a given path.
|
||||
:param path:
|
||||
|
||||
@@ -4,11 +4,11 @@ from typing import Optional, Tuple, List
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.schemas.search import SearchQuery, SearchItemType
|
||||
from basic_memory.services.search_service import SearchService
|
||||
|
||||
|
||||
class LinkResolver:
|
||||
@@ -58,7 +58,8 @@ class LinkResolver:
|
||||
logger.debug(
|
||||
f"Selected best match from {len(results)} results: {best_match.permalink}"
|
||||
)
|
||||
return await self.entity_repository.get_by_permalink(best_match.permalink)
|
||||
if best_match.permalink:
|
||||
return await self.entity_repository.get_by_permalink(best_match.permalink)
|
||||
|
||||
# if we couldn't find anything then return None
|
||||
return None
|
||||
@@ -106,9 +107,12 @@ class LinkResolver:
|
||||
score = result.score
|
||||
assert score is not None
|
||||
|
||||
# Parse path components
|
||||
path_parts = result.permalink.lower().split("/")
|
||||
last_part = path_parts[-1] if path_parts else ""
|
||||
if result.permalink:
|
||||
# Parse path components
|
||||
path_parts = result.permalink.lower().split("/")
|
||||
last_part = path_parts[-1] if path_parts else ""
|
||||
else:
|
||||
last_part = "" # pragma: no cover
|
||||
|
||||
# Title word match boosts
|
||||
term_matches = [term for term in terms if term in last_part]
|
||||
|
||||
@@ -128,10 +128,9 @@ class SearchService:
|
||||
self,
|
||||
entity: Entity,
|
||||
) -> None:
|
||||
|
||||
# delete all search index data associated with entity
|
||||
await self.repository.delete_by_entity_id(entity_id=entity.id)
|
||||
|
||||
|
||||
# reindex
|
||||
await self.index_entity_markdown(
|
||||
entity
|
||||
@@ -147,7 +146,6 @@ class SearchService:
|
||||
id=entity.id,
|
||||
type=SearchItemType.ENTITY.value,
|
||||
title=entity.title,
|
||||
permalink=entity.permalink,
|
||||
file_path=entity.file_path,
|
||||
metadata={
|
||||
"entity_type": entity.entity_type,
|
||||
@@ -179,6 +177,10 @@ class SearchService:
|
||||
Each type gets its own row in the search index with appropriate metadata.
|
||||
"""
|
||||
|
||||
assert entity.permalink is not None, (
|
||||
"entity.permalink should not be None for markdown entities"
|
||||
)
|
||||
|
||||
content_parts = []
|
||||
title_variants = self._generate_variants(entity.title)
|
||||
content_parts.extend(title_variants)
|
||||
@@ -192,6 +194,9 @@ class SearchService:
|
||||
|
||||
entity_content = "\n".join(p for p in content_parts if p and p.strip())
|
||||
|
||||
assert entity.permalink is not None, (
|
||||
"entity.permalink should not be None for markdown entities"
|
||||
)
|
||||
# Index entity
|
||||
await self.repository.index_item(
|
||||
SearchIndexRow(
|
||||
@@ -256,6 +261,10 @@ class SearchService:
|
||||
)
|
||||
)
|
||||
|
||||
async def delete_by_permalink(self, path_id: str):
|
||||
async def delete_by_permalink(self, permalink: str):
|
||||
"""Delete an item from the search index."""
|
||||
await self.repository.delete_by_permalink(path_id)
|
||||
await self.repository.delete_by_permalink(permalink)
|
||||
|
||||
async def delete_by_entity_id(self, entity_id: int):
|
||||
"""Delete an item from the search index."""
|
||||
await self.repository.delete_by_entity_id(entity_id)
|
||||
|
||||
@@ -3,4 +3,4 @@
|
||||
from .sync_service import SyncService
|
||||
from .watch_service import WatchService
|
||||
|
||||
__all__ = ["SyncService", "WatchService"]
|
||||
__all__ = ["SyncService", "WatchService"]
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
"""Service for syncing files between filesystem and database."""
|
||||
|
||||
import mimetypes
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Set, Dict, Sequence
|
||||
from typing import Set, Dict
|
||||
from typing import Tuple
|
||||
|
||||
import logfire
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.models import Entity
|
||||
@@ -44,7 +42,7 @@ class SyncReport:
|
||||
def total(self) -> int:
|
||||
"""Total number of changes."""
|
||||
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
|
||||
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
@@ -82,7 +80,7 @@ class SyncService:
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""Sync all files with database."""
|
||||
|
||||
with logfire.span(f"sync {directory}", directory=directory):
|
||||
with logfire.span(f"sync {directory}", directory=directory): # pyright: ignore [reportGeneralTypeIssues]
|
||||
# initial paths from db to sync
|
||||
# path -> checksum
|
||||
report = await self.scan(directory)
|
||||
@@ -91,7 +89,12 @@ class SyncService:
|
||||
|
||||
# sync moves first
|
||||
for old_path, new_path in report.moves.items():
|
||||
await self.handle_move(old_path, new_path)
|
||||
# in the case where a file has been deleted and replaced by another file
|
||||
# it will show up in the move and modified lists, so handle it in modified
|
||||
if new_path in report.modified:
|
||||
report.modified.remove(new_path)
|
||||
else:
|
||||
await self.handle_move(old_path, new_path)
|
||||
|
||||
# deleted next
|
||||
for path in report.deleted:
|
||||
@@ -109,23 +112,22 @@ class SyncService:
|
||||
|
||||
async def scan(self, directory):
|
||||
"""Scan directory for changes compared to database state."""
|
||||
|
||||
|
||||
db_paths = await self.get_db_file_state()
|
||||
|
||||
|
||||
# Track potentially moved files by checksum
|
||||
scan_result = await self.scan_directory(directory)
|
||||
report = SyncReport()
|
||||
|
||||
|
||||
# First find potential new files and record checksums
|
||||
# if a path is not present in the db, it could be new or could be the destination of a move
|
||||
for file_path, checksum in scan_result.files.items():
|
||||
if file_path not in db_paths:
|
||||
report.new.add(file_path)
|
||||
report.checksums[file_path] = checksum
|
||||
|
||||
|
||||
# Now detect moves and deletions
|
||||
for db_path, db_checksum in db_paths.items():
|
||||
|
||||
local_checksum_for_db_path = scan_result.files.get(db_path)
|
||||
|
||||
# file not modified
|
||||
@@ -135,7 +137,7 @@ class SyncService:
|
||||
# if checksums don't match for the same path, its modified
|
||||
if local_checksum_for_db_path and db_checksum != local_checksum_for_db_path:
|
||||
report.modified.add(db_path)
|
||||
report.checksums[db_path] = checksum
|
||||
report.checksums[db_path] = local_checksum_for_db_path
|
||||
|
||||
# check if it's moved or deleted
|
||||
if not local_checksum_for_db_path:
|
||||
@@ -143,9 +145,10 @@ class SyncService:
|
||||
if db_checksum in scan_result.checksums:
|
||||
new_path = scan_result.checksums[db_checksum]
|
||||
report.moves[db_path] = new_path
|
||||
|
||||
# Remove from new files since it's a move
|
||||
report.new.remove(new_path)
|
||||
|
||||
# Remove from new files if present
|
||||
if new_path in report.new:
|
||||
report.new.remove(new_path)
|
||||
|
||||
# deleted
|
||||
else:
|
||||
@@ -174,7 +177,7 @@ class SyncService:
|
||||
await self.search_service.index_entity(entity)
|
||||
return entity, checksum
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to sync {path}: {e}")
|
||||
raise
|
||||
|
||||
@@ -219,7 +222,7 @@ class SyncService:
|
||||
checksum = await self.file_service.compute_checksum(path)
|
||||
if new:
|
||||
# Generate permalink from path
|
||||
permalink = await self.entity_service.resolve_permalink(path)
|
||||
await self.entity_service.resolve_permalink(path)
|
||||
|
||||
# get file timestamps
|
||||
file_stats = self.file_service.file_stats(path)
|
||||
@@ -234,7 +237,6 @@ class SyncService:
|
||||
Entity(
|
||||
entity_type="file",
|
||||
file_path=path,
|
||||
permalink=permalink,
|
||||
checksum=checksum,
|
||||
title=file_path.name,
|
||||
created_at=created,
|
||||
@@ -242,13 +244,15 @@ class SyncService:
|
||||
content_type=content_type,
|
||||
)
|
||||
)
|
||||
return entity, checksum
|
||||
else:
|
||||
entity = await self.entity_repository.get_by_file_path(path)
|
||||
entity = await self.entity_repository.update(
|
||||
assert entity is not None, "entity should not be None for existing file"
|
||||
updated = await self.entity_repository.update(
|
||||
entity.id, {"file_path": path, "checksum": checksum}
|
||||
)
|
||||
|
||||
return entity, checksum
|
||||
assert updated is not None, "entity should be updated"
|
||||
return updated, checksum
|
||||
|
||||
async def handle_delete(self, file_path: str):
|
||||
"""Handle complete entity deletion including search index cleanup."""
|
||||
@@ -269,7 +273,10 @@ class SyncService:
|
||||
)
|
||||
logger.debug(f"Deleting from search index: {permalinks}")
|
||||
for permalink in permalinks:
|
||||
await self.search_service.delete_by_permalink(permalink)
|
||||
if permalink:
|
||||
await self.search_service.delete_by_permalink(permalink)
|
||||
else:
|
||||
await self.search_service.delete_by_entity_id(entity.id)
|
||||
|
||||
async def handle_move(self, old_path, new_path):
|
||||
logger.debug(f"Moving entity: {old_path} -> {new_path}")
|
||||
@@ -277,14 +284,16 @@ class SyncService:
|
||||
if entity:
|
||||
# Update file_path but keep the same permalink for link stability
|
||||
updated = await self.entity_repository.update(entity.id, {"file_path": new_path})
|
||||
assert updated is not None, "entity should be updated"
|
||||
# update search index
|
||||
await self.search_service.index_entity(updated)
|
||||
|
||||
async def resolve_relations(self):
|
||||
"""Try to resolve any unresolved relations"""
|
||||
|
||||
logger.debug("Attempting to resolve forward references")
|
||||
for relation in await self.relation_repository.find_unresolved_relations():
|
||||
unresolved_relations = await self.relation_repository.find_unresolved_relations()
|
||||
logger.debug(f"Attempting to resolve {len(unresolved_relations)} forward references")
|
||||
for relation in unresolved_relations:
|
||||
resolved_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
|
||||
|
||||
# ignore reference to self
|
||||
@@ -292,16 +301,13 @@ class SyncService:
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {resolved_entity.title}"
|
||||
)
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": resolved_entity.id,
|
||||
"to_name": resolved_entity.title,
|
||||
},
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": resolved_entity.id,
|
||||
"to_name": resolved_entity.title,
|
||||
},
|
||||
)
|
||||
|
||||
# update search index
|
||||
await self.search_service.index_entity(resolved_entity)
|
||||
@@ -320,17 +326,13 @@ class SyncService:
|
||||
logger.debug(f"Scanning directory: {directory}")
|
||||
result = ScanResult()
|
||||
|
||||
if not directory.exists():
|
||||
logger.debug(f"Directory does not exist: {directory}")
|
||||
return result
|
||||
|
||||
for root, dirnames, filenames in os.walk(str(directory)):
|
||||
# Skip dot directories in-place
|
||||
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
|
||||
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
|
||||
|
||||
for filename in filenames:
|
||||
# Skip dot files
|
||||
if filename.startswith('.'):
|
||||
if filename.startswith("."):
|
||||
continue
|
||||
|
||||
path = Path(root) / filename
|
||||
@@ -340,5 +342,4 @@ class SyncService:
|
||||
result.checksums[checksum] = rel_path
|
||||
logger.debug(f"Found file: {rel_path} with checksum: {checksum}")
|
||||
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
@@ -1,22 +1,20 @@
|
||||
"""Watch service for Basic Memory."""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import BaseModel
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from rich.console import Console
|
||||
from rich.live import Live
|
||||
from rich.table import Table
|
||||
from watchfiles import awatch, Change
|
||||
import os
|
||||
from watchfiles import awatch
|
||||
from watchfiles.main import FileChange, Change
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.sync.sync_service import SyncService
|
||||
from basic_memory.services.file_service import FileService
|
||||
from basic_memory.sync.sync_service import SyncService
|
||||
|
||||
|
||||
class WatchEvent(BaseModel):
|
||||
@@ -81,138 +79,136 @@ class WatchService:
|
||||
self.status_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.console = Console()
|
||||
|
||||
def generate_table(self) -> Table:
|
||||
"""Generate status display table"""
|
||||
table = Table()
|
||||
|
||||
# Add status row
|
||||
table.add_column("Status", style="cyan")
|
||||
table.add_column("Last Scan", style="cyan")
|
||||
table.add_column("Files", style="cyan")
|
||||
table.add_column("Errors", style="red")
|
||||
|
||||
# Add main status row
|
||||
table.add_row(
|
||||
"✓ Running" if self.state.running else "✗ Stopped",
|
||||
self.state.last_scan.strftime("%H:%M:%S") if self.state.last_scan else "-",
|
||||
str(self.state.synced_files),
|
||||
f"{self.state.error_count} ({self.state.last_error.strftime('%H:%M:%S') if self.state.last_error else 'none'})",
|
||||
)
|
||||
|
||||
if self.state.recent_events:
|
||||
# Add recent events
|
||||
table.add_section()
|
||||
table.add_row("Recent Events", "", "", "")
|
||||
|
||||
for event in self.state.recent_events[:5]: # Show last 5 events
|
||||
color = {
|
||||
"new": "green",
|
||||
"modified": "yellow",
|
||||
"moved": "blue",
|
||||
"deleted": "red",
|
||||
"error": "red",
|
||||
}.get(event.action, "white")
|
||||
|
||||
icon = {
|
||||
"new": "✚",
|
||||
"modified": "✎",
|
||||
"moved": "→",
|
||||
"deleted": "✖",
|
||||
"error": "!",
|
||||
}.get(event.action, "*")
|
||||
|
||||
table.add_row(
|
||||
f"[{color}]{icon} {event.action}[/{color}]",
|
||||
event.timestamp.strftime("%H:%M:%S"),
|
||||
f"[{color}]{event.path}[/{color}]",
|
||||
f"[dim]{event.checksum[:8] if event.checksum else ''}[/dim]",
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
async def run(self, console_status: bool = False): # pragma: no cover
|
||||
async def run(self): # pragma: no cover
|
||||
"""Watch for file changes and sync them"""
|
||||
logger.info("Watching for sync changes")
|
||||
self.state.running = True
|
||||
self.state.start_time = datetime.now()
|
||||
await self.write_status()
|
||||
try:
|
||||
async for changes in awatch(
|
||||
self.config.home,
|
||||
debounce=self.config.sync_delay,
|
||||
watch_filter=self.filter_changes,
|
||||
recursive=True,
|
||||
):
|
||||
await self.handle_changes(self.config.home, changes)
|
||||
|
||||
if console_status:
|
||||
with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
|
||||
try:
|
||||
async for changes in awatch(
|
||||
self.config.home,
|
||||
watch_filter=self.filter_changes,
|
||||
debounce=self.config.sync_delay,
|
||||
recursive=True,
|
||||
):
|
||||
# Process changes
|
||||
await self.handle_changes(self.config.home)
|
||||
# Update display
|
||||
live.update(self.generate_table())
|
||||
except Exception as e:
|
||||
self.state.record_error(str(e))
|
||||
await self.write_status()
|
||||
raise
|
||||
finally:
|
||||
self.state.running = False
|
||||
await self.write_status()
|
||||
|
||||
except Exception as e:
|
||||
self.state.record_error(str(e))
|
||||
await self.write_status()
|
||||
raise
|
||||
finally:
|
||||
self.state.running = False
|
||||
await self.write_status()
|
||||
def filter_changes(self, change: Change, path: str) -> bool:
|
||||
"""Filter to only watch non-hidden files and directories.
|
||||
|
||||
else:
|
||||
try:
|
||||
async for changes in awatch(
|
||||
self.config.home,
|
||||
watch_filter=self.filter_changes,
|
||||
debounce=self.config.sync_delay,
|
||||
recursive=True,
|
||||
):
|
||||
# Process changes
|
||||
await self.handle_changes(self.config.home)
|
||||
# Update display
|
||||
Returns:
|
||||
True if the file should be watched, False if it should be ignored
|
||||
"""
|
||||
# Skip if path is invalid
|
||||
try:
|
||||
relative_path = Path(path).relative_to(self.config.home)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
self.state.record_error(str(e))
|
||||
await self.write_status()
|
||||
raise
|
||||
finally:
|
||||
self.state.running = False
|
||||
await self.write_status()
|
||||
# Skip hidden directories and files
|
||||
path_parts = relative_path.parts
|
||||
for part in path_parts:
|
||||
if part.startswith("."):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
async def write_status(self):
|
||||
"""Write current state to status file"""
|
||||
self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
|
||||
|
||||
def filter_changes(self, change: Change, path: str) -> bool:
|
||||
"""Filter to only watch markdown files"""
|
||||
return path.endswith(".md") and not Path(path).name.startswith(".")
|
||||
|
||||
async def handle_changes(self, directory: Path):
|
||||
async def handle_changes(self, directory: Path, changes: Set[FileChange]):
|
||||
"""Process a batch of file changes"""
|
||||
logger.debug(f"handling {len(changes)} changes in directory: {directory} ...")
|
||||
|
||||
# Group changes by type
|
||||
adds = []
|
||||
deletes = []
|
||||
modifies = []
|
||||
|
||||
for change, path in changes:
|
||||
# convert to relative path
|
||||
relative_path = str(Path(path).relative_to(directory))
|
||||
if change == Change.added:
|
||||
adds.append(relative_path)
|
||||
elif change == Change.deleted:
|
||||
deletes.append(relative_path)
|
||||
elif change == Change.modified:
|
||||
modifies.append(relative_path)
|
||||
|
||||
# Track processed files to avoid duplicates
|
||||
processed = set()
|
||||
|
||||
# First handle potential moves
|
||||
for added_path in adds:
|
||||
if added_path in processed:
|
||||
continue # pragma: no cover
|
||||
|
||||
for deleted_path in deletes:
|
||||
if deleted_path in processed:
|
||||
continue # pragma: no cover
|
||||
|
||||
if added_path != deleted_path:
|
||||
# Compare checksums to detect moves
|
||||
try:
|
||||
added_checksum = await self.file_service.compute_checksum(added_path)
|
||||
deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
|
||||
deleted_path
|
||||
)
|
||||
|
||||
if deleted_entity and deleted_entity.checksum == added_checksum:
|
||||
await self.sync_service.handle_move(deleted_path, added_path)
|
||||
self.state.add_event(
|
||||
path=f"{deleted_path} -> {added_path}",
|
||||
action="moved",
|
||||
status="success",
|
||||
)
|
||||
self.console.print(
|
||||
f"[blue]→[/blue] Moved: {deleted_path} → {added_path}"
|
||||
)
|
||||
processed.add(added_path)
|
||||
processed.add(deleted_path)
|
||||
break
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.warning(f"Error checking for move: {e}")
|
||||
|
||||
# Handle remaining changes
|
||||
for path in deletes:
|
||||
if path not in processed:
|
||||
await self.sync_service.handle_delete(path)
|
||||
self.state.add_event(path=path, action="deleted", status="success")
|
||||
self.console.print(f"[red]✕[/red] Deleted: {path}")
|
||||
processed.add(path)
|
||||
|
||||
for path in adds:
|
||||
if path not in processed:
|
||||
_, checksum = await self.sync_service.sync_file(path, new=True)
|
||||
self.state.add_event(path=path, action="new", status="success", checksum=checksum)
|
||||
self.console.print(f"[green]✓[/green] Added: {path}")
|
||||
processed.add(path)
|
||||
|
||||
for path in modifies:
|
||||
if path not in processed:
|
||||
_, checksum = await self.sync_service.sync_file(path, new=False)
|
||||
self.state.add_event(
|
||||
path=path, action="modified", status="success", checksum=checksum
|
||||
)
|
||||
self.console.print(f"[yellow]✎[/yellow] Modified: {path}")
|
||||
processed.add(path)
|
||||
|
||||
# Add a divider if we processed any files
|
||||
if processed:
|
||||
self.console.print("─" * 50, style="dim")
|
||||
|
||||
logger.debug(f"handling change in directory: {directory} ...")
|
||||
# Process changes with timeout
|
||||
report = await self.sync_service.sync(directory)
|
||||
self.state.last_scan = datetime.now()
|
||||
self.state.synced_files = report.total
|
||||
|
||||
# Update stats
|
||||
for path in report.new:
|
||||
self.state.add_event(
|
||||
path=path, action="new", status="success", checksum=report.checksums[path]
|
||||
)
|
||||
for path in report.modified:
|
||||
self.state.add_event(
|
||||
path=path, action="modified", status="success", checksum=report.checksums[path]
|
||||
)
|
||||
for old_path, new_path in report.moves.items():
|
||||
self.state.add_event(
|
||||
path=f"{old_path} -> {new_path}",
|
||||
action="moved",
|
||||
status="success",
|
||||
checksum=report.checksums[new_path],
|
||||
)
|
||||
for path in report.deleted:
|
||||
self.state.add_event(path=path, action="deleted", status="success")
|
||||
|
||||
self.state.synced_files += len(processed)
|
||||
await self.write_status()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Utility functions for basic-memory."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -64,8 +65,12 @@ def generate_permalink(file_path: Union[Path, str]) -> str:
|
||||
|
||||
|
||||
def setup_logging(
|
||||
env: str, home_dir: Path, log_file: Optional[str] = None, log_level: str = "INFO", console: bool = True
|
||||
, ) -> None: # pragma: no cover
|
||||
env: str,
|
||||
home_dir: Path,
|
||||
log_file: Optional[str] = None,
|
||||
log_level: str = "INFO",
|
||||
console: bool = True,
|
||||
) -> None: # pragma: no cover
|
||||
"""
|
||||
Configure logging for the application.
|
||||
:param home_dir: the root directory for the application
|
||||
@@ -116,4 +121,7 @@ def setup_logging(
|
||||
# Get the logger for 'httpx'
|
||||
httpx_logger = logging.getLogger("httpx")
|
||||
# Set the logging level to WARNING to ignore INFO and DEBUG logs
|
||||
httpx_logger.setLevel(logging.WARNING)
|
||||
httpx_logger.setLevel(logging.WARNING)
|
||||
|
||||
# turn watchfiles to WARNING
|
||||
logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
|
||||
|
||||
@@ -10,12 +10,21 @@ from basic_memory.cli.commands.status import (
|
||||
group_changes_by_directory,
|
||||
display_changes,
|
||||
)
|
||||
from basic_memory.config import config
|
||||
from basic_memory.sync.sync_service import SyncReport
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_status_command(tmp_path, monkeypatch):
|
||||
"""Test CLI status command."""
|
||||
config.home = tmp_path
|
||||
# Should exit with code 0
|
||||
result = runner.invoke(app, ["status", "--verbose"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_status_command_error(tmp_path, monkeypatch):
|
||||
"""Test CLI status command error handling."""
|
||||
@@ -117,4 +126,4 @@ def test_add_files_to_tree():
|
||||
checksums = {"dir1/file1.md": "abcd1234", "dir1/file2.md": "efgh5678"}
|
||||
|
||||
tree = Tree("Test with checksums")
|
||||
add_files_to_tree(tree, paths, "green", checksums)
|
||||
add_files_to_tree(tree, paths, "green", checksums)
|
||||
|
||||
@@ -107,4 +107,4 @@ async def test_run_sync_watch_mode(sync_service, test_config):
|
||||
def test_sync_command():
|
||||
"""Test the sync command."""
|
||||
result = runner.invoke(app, ["sync", "--verbose"])
|
||||
assert result.exit_code == 0
|
||||
assert result.exit_code == 0
|
||||
|
||||
+38
-3
@@ -1,5 +1,6 @@
|
||||
"""Common test fixtures."""
|
||||
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
from typing import AsyncGenerator
|
||||
from datetime import datetime, timezone
|
||||
@@ -137,8 +138,6 @@ def entity_parser(test_config):
|
||||
return EntityParser(test_config.home)
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def sync_service(
|
||||
entity_service: EntityService,
|
||||
@@ -315,4 +314,40 @@ async def test_graph(
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def watch_service(sync_service, file_service, test_config):
|
||||
return WatchService(sync_service=sync_service, file_service=file_service, config=test_config)
|
||||
return WatchService(sync_service=sync_service, file_service=file_service, config=test_config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_files(test_config) -> dict[str, Path]:
|
||||
"""Copy test files into the project directory.
|
||||
|
||||
Returns a dict mapping file names to their paths in the project dir.
|
||||
"""
|
||||
# Source files relative to tests directory
|
||||
source_files = {
|
||||
"pdf": Path("tests/Non-MarkdownFileSupport.pdf"),
|
||||
"image": Path("tests/Screenshot.png"),
|
||||
}
|
||||
|
||||
# Create copies in temp project directory
|
||||
project_files = {}
|
||||
for name, src_path in source_files.items():
|
||||
# Read source file
|
||||
content = src_path.read_bytes()
|
||||
|
||||
# Create destination path and ensure parent dirs exist
|
||||
dest_path = test_config.home / src_path.name
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write file
|
||||
dest_path.write_bytes(content)
|
||||
project_files[name] = dest_path
|
||||
|
||||
return project_files
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def synced_files(sync_service, test_config, test_files):
|
||||
# Initial sync - should create forward reference
|
||||
await sync_service.sync(test_config.home)
|
||||
return test_files
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Tests for resource tools that exercise the full stack with SQLite."""
|
||||
|
||||
import io
|
||||
import base64
|
||||
from PIL import Image as PILImage
|
||||
|
||||
import pytest
|
||||
from mcp.server.fastmcp.exceptions import ToolError
|
||||
|
||||
from basic_memory.mcp.tools import resource
|
||||
from basic_memory.mcp.tools import notes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_text_file(app, synced_files):
|
||||
"""Test reading a text file.
|
||||
|
||||
Should:
|
||||
- Correctly identify text content
|
||||
- Return the content as text
|
||||
- Include correct metadata
|
||||
"""
|
||||
# First create a text file via notes
|
||||
result = await notes.write_note(
|
||||
title="Text Resource",
|
||||
folder="test",
|
||||
content="This is a test text resource",
|
||||
tags=["test", "resource"],
|
||||
)
|
||||
assert result is not None
|
||||
|
||||
# Now read it as a resource
|
||||
response = await resource.read_resource("test/text-resource")
|
||||
|
||||
assert response["type"] == "text"
|
||||
assert "This is a test text resource" in response["text"]
|
||||
assert response["content_type"].startswith("text/")
|
||||
assert response["encoding"] == "utf-8"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_image_file(app, synced_files):
|
||||
"""Test reading an image file.
|
||||
|
||||
Should:
|
||||
- Correctly identify image content
|
||||
- Optimize the image
|
||||
- Return base64 encoded image data
|
||||
"""
|
||||
# Get the path to the synced image file
|
||||
image_path = synced_files["image"].name
|
||||
|
||||
# Read it as a resource
|
||||
response = await resource.read_resource(image_path)
|
||||
|
||||
assert response["type"] == "image"
|
||||
assert response["source"]["type"] == "base64"
|
||||
assert response["source"]["media_type"] == "image/jpeg"
|
||||
|
||||
# Verify the image data is valid base64 that can be decoded
|
||||
img_data = base64.b64decode(response["source"]["data"])
|
||||
assert len(img_data) > 0
|
||||
|
||||
# Should be able to open as an image
|
||||
img = PILImage.open(io.BytesIO(img_data))
|
||||
assert img.width > 0
|
||||
assert img.height > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_pdf_file(app, synced_files):
|
||||
"""Test reading a PDF file.
|
||||
|
||||
Should:
|
||||
- Correctly identify PDF content
|
||||
- Return base64 encoded PDF data
|
||||
"""
|
||||
# Get the path to the synced PDF file
|
||||
pdf_path = synced_files["pdf"].name
|
||||
|
||||
# Read it as a resource
|
||||
response = await resource.read_resource(pdf_path)
|
||||
|
||||
assert response["type"] == "document"
|
||||
assert response["source"]["type"] == "base64"
|
||||
assert response["source"]["media_type"] == "application/pdf"
|
||||
|
||||
# Verify the PDF data is valid base64 that can be decoded
|
||||
pdf_data = base64.b64decode(response["source"]["data"])
|
||||
assert len(pdf_data) > 0
|
||||
assert pdf_data.startswith(b"%PDF") # PDF signature
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_not_found(app):
|
||||
"""Test trying to read a non-existent resource."""
|
||||
with pytest.raises(ToolError, match="Error calling tool: Client error '404 Not Found'"):
|
||||
await resource.read_resource("does-not-exist")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_memory_url(app, synced_files):
|
||||
"""Test reading a resource using a memory:// URL."""
|
||||
# Create a text file via notes
|
||||
await notes.write_note(
|
||||
title="Memory URL Test",
|
||||
folder="test",
|
||||
content="Testing memory:// URL handling for resources",
|
||||
)
|
||||
|
||||
# Read it with a memory:// URL
|
||||
memory_url = "memory://test/memory-url-test"
|
||||
response = await resource.read_resource(memory_url)
|
||||
|
||||
assert response["type"] == "text"
|
||||
assert "Testing memory:// URL handling for resources" in response["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_image_optimization_functions(app):
|
||||
"""Test the image optimization helper functions."""
|
||||
# Create a test image
|
||||
img = PILImage.new("RGB", (1000, 800), color="white")
|
||||
|
||||
# Test calculate_target_params function
|
||||
# Small image
|
||||
quality, size = resource.calculate_target_params(100000)
|
||||
assert quality == 70
|
||||
assert size == 1000
|
||||
|
||||
# Medium image
|
||||
quality, size = resource.calculate_target_params(800000)
|
||||
assert quality == 60
|
||||
assert size == 800
|
||||
|
||||
# Large image
|
||||
quality, size = resource.calculate_target_params(2000000)
|
||||
assert quality == 50
|
||||
assert size == 600
|
||||
|
||||
# Test resize_image function
|
||||
# Image that needs resizing
|
||||
resized = resource.resize_image(img, 500)
|
||||
assert resized.width <= 500
|
||||
assert resized.height <= 500
|
||||
|
||||
# Image that doesn't need resizing
|
||||
small_img = PILImage.new("RGB", (300, 200), color="white")
|
||||
resized = resource.resize_image(small_img, 500)
|
||||
assert resized.width == 300
|
||||
assert resized.height == 200
|
||||
|
||||
# Test optimize_image function
|
||||
img_bytes = io.BytesIO()
|
||||
img.save(img_bytes, format="PNG")
|
||||
img_bytes.seek(0)
|
||||
content_length = len(img_bytes.getvalue())
|
||||
|
||||
# In a small test image, optimization might make the image larger
|
||||
# because of JPEG overhead. Let's just test that it returns something
|
||||
optimized = resource.optimize_image(img, content_length)
|
||||
assert len(optimized) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_with_transparency(app, synced_files, mocker):
|
||||
"""Test reading an image with transparency.
|
||||
|
||||
Should:
|
||||
- Convert RGBA images to RGB
|
||||
- Handle transparency correctly
|
||||
"""
|
||||
# Mock the response to simulate an RGBA image
|
||||
mock_response = mocker.MagicMock()
|
||||
mock_response.headers = {"content-type": "image/png", "content-length": "10000"}
|
||||
|
||||
# Create a test PNG with transparency
|
||||
img = PILImage.new("RGBA", (500, 400), color=(255, 255, 255, 0))
|
||||
img_bytes = io.BytesIO()
|
||||
img.save(img_bytes, format="PNG")
|
||||
img_bytes.seek(0)
|
||||
mock_response.content = img_bytes.getvalue()
|
||||
|
||||
# Mock call_get to return our transparent image
|
||||
mocker.patch("basic_memory.mcp.tools.resource.call_get", return_value=mock_response)
|
||||
|
||||
# Test reading the resource
|
||||
response = await resource.read_resource("transparent-image.png")
|
||||
|
||||
assert response["type"] == "image"
|
||||
assert response["source"]["media_type"] == "image/jpeg"
|
||||
|
||||
# Verify the image data is valid and was converted to RGB
|
||||
img_data = base64.b64decode(response["source"]["data"])
|
||||
img = PILImage.open(io.BytesIO(img_data))
|
||||
assert img.mode == "RGB" # Should be converted from RGBA to RGB
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_large_document(app, mocker):
|
||||
"""Test handling of documents that exceed the size limit.
|
||||
|
||||
Should:
|
||||
- Detect when document size exceeds limit
|
||||
- Return appropriate error message
|
||||
"""
|
||||
# Mock the response to simulate a large document
|
||||
mock_response = mocker.MagicMock()
|
||||
mock_response.headers = {"content-type": "application/octet-stream", "content-length": "500000"}
|
||||
mock_response.content = b"0" * 500000 # Create a large fake binary document
|
||||
|
||||
# Mock call_get to return our large document
|
||||
mocker.patch("basic_memory.mcp.tools.resource.call_get", return_value=mock_response)
|
||||
|
||||
# Test reading the resource
|
||||
response = await resource.read_resource("large-document.bin")
|
||||
|
||||
assert response["type"] == "error"
|
||||
assert "Document size 500000 bytes exceeds maximum allowed size" in response["error"]
|
||||
|
||||
|
||||
# Let's skip the minimum parameters test since those values are internal to the optimize_image function
|
||||
# The rest of the code is well covered by the other tests
|
||||
# @pytest.mark.skip("Minimum parameter test not needed - code already has good coverage")
|
||||
# @pytest.mark.asyncio
|
||||
# async def test_optimize_image_limits(app, monkeypatch):
|
||||
# """Test image optimization when it reaches minimum parameters."""
|
||||
# pass
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Tests for MemoryUrl parsing."""
|
||||
|
||||
from basic_memory.schemas.memory import memory_url, memory_url_path
|
||||
from basic_memory.schemas.memory import memory_url, memory_url_path, normalize_memory_url
|
||||
|
||||
|
||||
def test_basic_permalink():
|
||||
@@ -44,3 +44,20 @@ def test_str_representation():
|
||||
"""Test converting back to string."""
|
||||
url = memory_url.validate_python("memory://specs/search")
|
||||
assert url == "memory://specs/search"
|
||||
|
||||
|
||||
def test_normalize_memory_url():
|
||||
"""Test converting back to string."""
|
||||
url = normalize_memory_url("memory://specs/search")
|
||||
assert url == "memory://specs/search"
|
||||
|
||||
|
||||
def test_normalize_memory_url_no_prefix():
|
||||
"""Test converting back to string."""
|
||||
url = normalize_memory_url("specs/search")
|
||||
assert url == "memory://specs/search"
|
||||
|
||||
|
||||
def test_normalize_memory_url_empty():
|
||||
"""Test converting back to string."""
|
||||
assert normalize_memory_url("") == ""
|
||||
|
||||
@@ -144,4 +144,4 @@ async def test_context_metadata(context_service, test_graph):
|
||||
assert metadata["uri"] == "test/root"
|
||||
assert metadata["depth"] == 2
|
||||
assert metadata["generated_at"] is not None
|
||||
assert metadata["matched_results"] > 0
|
||||
assert metadata["matched_results"] > 0
|
||||
|
||||
@@ -184,6 +184,25 @@ async def test_delete_entity_success(entity_service: EntityService):
|
||||
await entity_service.get_by_permalink(entity_data.permalink)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_entity_by_id(entity_service: EntityService):
|
||||
"""Test successful entity deletion."""
|
||||
entity_data = EntitySchema(
|
||||
title="TestEntity",
|
||||
folder="test",
|
||||
entity_type="test",
|
||||
)
|
||||
created = await entity_service.create_entity(entity_data)
|
||||
|
||||
# Act using permalink
|
||||
result = await entity_service.delete_entity(created.id)
|
||||
|
||||
# Assert
|
||||
assert result is True
|
||||
with pytest.raises(EntityNotFoundError):
|
||||
await entity_service.get_by_permalink(entity_data.permalink)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_entity_by_permalink_not_found(entity_service: EntityService):
|
||||
"""Test handling of non-existent entity retrieval."""
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
"""Tests for link resolution service."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
import pytest_asyncio
|
||||
|
||||
from basic_memory.schemas.base import Entity as EntitySchema
|
||||
from basic_memory.services.link_resolver import LinkResolver
|
||||
from basic_memory.models.knowledge import Entity as EntityModel
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@@ -65,7 +68,19 @@ async def test_entities(entity_service, file_service):
|
||||
)
|
||||
)
|
||||
|
||||
return [e1, e2, e3, e4]
|
||||
# non markdown entity
|
||||
e7 = await entity_service.repository.add(
|
||||
EntityModel(
|
||||
title="Image.png",
|
||||
entity_type="file",
|
||||
content_type="image/png",
|
||||
file_path="Image.png",
|
||||
created_at=datetime.now(timezone.utc),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
|
||||
return [e1, e2, e3, e4, e5, e6, e7]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@@ -131,3 +146,13 @@ async def test_resolve_none(link_resolver):
|
||||
"""Test resolving non-existent entity."""
|
||||
# Basic new entity
|
||||
assert await link_resolver.resolve_link("New Feature") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_file(link_resolver):
|
||||
"""Test resolving non-existent entity."""
|
||||
# Basic new entity
|
||||
resolved = await link_resolver.resolve_link("Image.png")
|
||||
assert resolved is not None
|
||||
assert resolved.entity_type == "file"
|
||||
assert resolved.title == "Image.png"
|
||||
|
||||
+161
-35
@@ -15,34 +15,6 @@ from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.sync.sync_service import SyncService
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_files(test_config) -> dict[str, Path]:
|
||||
"""Copy test files into the project directory.
|
||||
|
||||
Returns a dict mapping file names to their paths in the project dir.
|
||||
"""
|
||||
# Source files relative to tests directory
|
||||
source_files = {
|
||||
"pdf": Path("tests/Non-MarkdownFileSupport.pdf"),
|
||||
"image": Path("tests/Screenshot.png")
|
||||
}
|
||||
|
||||
# Create copies in temp project directory
|
||||
project_files = {}
|
||||
for name, src_path in source_files.items():
|
||||
# Read source file
|
||||
content = src_path.read_bytes()
|
||||
|
||||
# Create destination path and ensure parent dirs exist
|
||||
dest_path = test_config.home / src_path.name
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write file
|
||||
dest_path.write_bytes(content)
|
||||
project_files[name] = dest_path
|
||||
|
||||
return project_files
|
||||
|
||||
async def create_test_file(path: Path, content: str = "test content") -> None:
|
||||
"""Create a test file with given content."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -162,6 +134,25 @@ A test concept.
|
||||
assert relations[0].to_name == "concept/other"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_hidden_file(
|
||||
sync_service: SyncService, test_config: ProjectConfig, entity_service: EntityService
|
||||
):
|
||||
"""Test basic knowledge sync functionality."""
|
||||
# Create test files
|
||||
project_dir = test_config.home
|
||||
|
||||
# hidden file
|
||||
await create_test_file(project_dir / "concept/.hidden.md", "hidden")
|
||||
|
||||
# Run sync
|
||||
await sync_service.sync(test_config.home)
|
||||
|
||||
# Verify results
|
||||
entities = await entity_service.repository.find_all()
|
||||
assert len(entities) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_entity_with_nonexistent_relations(
|
||||
sync_service: SyncService, test_config: ProjectConfig
|
||||
@@ -498,8 +489,8 @@ modified: 2024-01-01
|
||||
# Verify final state
|
||||
doc = await sync_service.entity_service.repository.get_by_permalink("changing")
|
||||
assert doc is not None
|
||||
|
||||
# if we failed in the middle of a sync, the next one should fix it.
|
||||
|
||||
# if we failed in the middle of a sync, the next one should fix it.
|
||||
if doc.checksum is None:
|
||||
await sync_service.sync(test_config.home)
|
||||
doc = await sync_service.entity_service.repository.get_by_permalink("changing")
|
||||
@@ -876,20 +867,155 @@ test content
|
||||
async def test_sync_non_markdown_files(sync_service, test_config, test_files):
|
||||
"""Test syncing non-markdown files."""
|
||||
report = await sync_service.sync(test_config.home)
|
||||
|
||||
assert report.total == 2
|
||||
|
||||
# Check files were detected
|
||||
assert test_files["pdf"].name in [f for f in report.new]
|
||||
assert test_files["image"].name in [f for f in report.new]
|
||||
|
||||
# Verify entities were created
|
||||
pdf_entity = await sync_service.entity_repository.get_by_file_path(
|
||||
str(test_files["pdf"].name)
|
||||
)
|
||||
pdf_entity = await sync_service.entity_repository.get_by_file_path(str(test_files["pdf"].name))
|
||||
assert pdf_entity is not None, "PDF entity should have been created"
|
||||
assert pdf_entity.content_type == "application/pdf"
|
||||
|
||||
image_entity = await sync_service.entity_repository.get_by_file_path(
|
||||
str(test_files["image"].name)
|
||||
)
|
||||
assert image_entity.content_type == "image/png"
|
||||
assert image_entity.content_type == "image/png"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_non_markdown_files_modified(
|
||||
sync_service, test_config, test_files, file_service
|
||||
):
|
||||
"""Test syncing non-markdown files."""
|
||||
report = await sync_service.sync(test_config.home)
|
||||
assert report.total == 2
|
||||
|
||||
# Check files were detected
|
||||
assert test_files["pdf"].name in [f for f in report.new]
|
||||
assert test_files["image"].name in [f for f in report.new]
|
||||
|
||||
test_files["pdf"].write_text("New content")
|
||||
test_files["image"].write_text("New content")
|
||||
|
||||
report = await sync_service.sync(test_config.home)
|
||||
assert len(report.modified) == 2
|
||||
|
||||
pdf_file_content, pdf_checksum = await file_service.read_file(test_files["pdf"].name)
|
||||
image_file_content, img_checksum = await file_service.read_file(test_files["image"].name)
|
||||
|
||||
pdf_entity = await sync_service.entity_repository.get_by_file_path(str(test_files["pdf"].name))
|
||||
image_entity = await sync_service.entity_repository.get_by_file_path(
|
||||
str(test_files["image"].name)
|
||||
)
|
||||
|
||||
assert pdf_entity.checksum == pdf_checksum
|
||||
assert image_entity.checksum == img_checksum
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_non_markdown_files_move(sync_service, test_config, test_files):
|
||||
"""Test syncing non-markdown files updates permalink"""
|
||||
report = await sync_service.sync(test_config.home)
|
||||
assert report.total == 2
|
||||
|
||||
# Check files were detected
|
||||
assert test_files["pdf"].name in [f for f in report.new]
|
||||
assert test_files["image"].name in [f for f in report.new]
|
||||
|
||||
test_files["pdf"].rename(test_config.home / "moved_pdf.pdf")
|
||||
report2 = await sync_service.sync(test_config.home)
|
||||
assert len(report2.moves) == 1
|
||||
|
||||
# Verify entity is updated
|
||||
pdf_entity = await sync_service.entity_repository.get_by_file_path("moved_pdf.pdf")
|
||||
assert pdf_entity is not None
|
||||
assert pdf_entity.permalink is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_non_markdown_files_deleted(sync_service, test_config, test_files):
|
||||
"""Test syncing non-markdown files updates permalink"""
|
||||
report = await sync_service.sync(test_config.home)
|
||||
assert report.total == 2
|
||||
|
||||
# Check files were detected
|
||||
assert test_files["pdf"].name in [f for f in report.new]
|
||||
assert test_files["image"].name in [f for f in report.new]
|
||||
|
||||
test_files["pdf"].unlink()
|
||||
report2 = await sync_service.sync(test_config.home)
|
||||
assert len(report2.deleted) == 1
|
||||
|
||||
# Verify entity is deleted
|
||||
pdf_entity = await sync_service.entity_repository.get_by_file_path("moved_pdf.pdf")
|
||||
assert pdf_entity is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_non_markdown_files_move_with_delete(
|
||||
sync_service, test_config, test_files, file_service
|
||||
):
|
||||
"""Test syncing non-markdown files handles file deletes and renames during sync"""
|
||||
|
||||
# Create initial files
|
||||
await create_test_file(test_config.home / "doc.pdf", "content1")
|
||||
await create_test_file(test_config.home / "other/doc-1.pdf", "content2")
|
||||
|
||||
# Initial sync
|
||||
await sync_service.sync(test_config.home)
|
||||
|
||||
# First move/delete the original file to make way for the move
|
||||
(test_config.home / "doc.pdf").unlink()
|
||||
(test_config.home / "other/doc-1.pdf").rename(test_config.home / "doc.pdf")
|
||||
|
||||
# Sync again
|
||||
await sync_service.sync(test_config.home)
|
||||
|
||||
# Verify the changes
|
||||
moved_entity = await sync_service.entity_repository.get_by_file_path("doc.pdf")
|
||||
assert moved_entity is not None
|
||||
assert moved_entity.permalink is None
|
||||
|
||||
file_content, _ = await file_service.read_file("doc.pdf")
|
||||
assert "content2" in file_content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_relation_to_non_markdown_file(
|
||||
sync_service: SyncService, test_config: ProjectConfig, file_service: FileService, test_files
|
||||
):
|
||||
"""Test that sync resolves permalink conflicts on update."""
|
||||
project_dir = test_config.home
|
||||
|
||||
content = f"""
|
||||
---
|
||||
title: a note
|
||||
type: note
|
||||
tags: []
|
||||
---
|
||||
|
||||
- relates_to [[{test_files["pdf"].name}]]
|
||||
"""
|
||||
|
||||
note_file = project_dir / "note.md"
|
||||
await create_test_file(note_file, content)
|
||||
|
||||
# Run sync
|
||||
await sync_service.sync(test_config.home)
|
||||
|
||||
# Check permalinks
|
||||
file_one_content, _ = await file_service.read_file(note_file)
|
||||
assert (
|
||||
f"""---
|
||||
title: a note
|
||||
type: note
|
||||
tags: []
|
||||
permalink: note
|
||||
---
|
||||
|
||||
- relates_to [[{test_files["pdf"].name}]]
|
||||
""".strip()
|
||||
== file_one_content
|
||||
)
|
||||
|
||||
@@ -1,40 +1,25 @@
|
||||
"""Tests for watch service."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from watchfiles import Change
|
||||
|
||||
from basic_memory.services.file_service import FileService
|
||||
from basic_memory.sync.sync_service import SyncReport
|
||||
from basic_memory.sync.sync_service import SyncService
|
||||
from basic_memory.sync.watch_service import WatchService, WatchServiceState
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sync_service(mocker):
|
||||
"""Create mock sync service."""
|
||||
service = mocker.Mock(spec=SyncService)
|
||||
service.sync.return_value = SyncReport(
|
||||
new={"test.md"},
|
||||
modified={"modified.md"},
|
||||
deleted={"deleted.md"},
|
||||
moves={"old.md": "new.md"},
|
||||
checksums={"test.md": "abcd1234", "modified.md": "efgh5678", "new.md": "ijkl9012"},
|
||||
)
|
||||
return service
|
||||
async def create_test_file(path: Path, content: str = "test content") -> None:
|
||||
"""Create a test file with given content."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_file_service(mocker):
|
||||
"""Create mock file service."""
|
||||
return mocker.Mock(spec=FileService)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def watch_service(mock_sync_service, mock_file_service, test_config):
|
||||
def watch_service(sync_service, file_service, test_config):
|
||||
"""Create watch service instance."""
|
||||
return WatchService(mock_sync_service, mock_file_service, test_config)
|
||||
return WatchService(sync_service, file_service, test_config)
|
||||
|
||||
|
||||
def test_watch_service_init(watch_service, test_config):
|
||||
@@ -42,14 +27,6 @@ def test_watch_service_init(watch_service, test_config):
|
||||
assert watch_service.status_path.parent.exists()
|
||||
|
||||
|
||||
def test_filter_changes(watch_service):
|
||||
"""Test file change filtering."""
|
||||
assert watch_service.filter_changes(Change.added, "test.md")
|
||||
assert watch_service.filter_changes(Change.modified, "dir/test.md")
|
||||
assert not watch_service.filter_changes(Change.added, "test.txt")
|
||||
assert not watch_service.filter_changes(Change.added, ".hidden.md")
|
||||
|
||||
|
||||
def test_state_add_event():
|
||||
"""Test adding events to state."""
|
||||
state = WatchServiceState()
|
||||
@@ -91,32 +68,296 @@ async def test_write_status(watch_service):
|
||||
assert data["error_count"] == 0
|
||||
|
||||
|
||||
def test_generate_table(watch_service):
|
||||
"""Test status table generation."""
|
||||
# Add some test events
|
||||
watch_service.state.add_event("test.md", "new", "success", "abcd1234")
|
||||
watch_service.state.add_event("modified.md", "modified", "success", "efgh5678")
|
||||
watch_service.state.record_error("test error")
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_file_add(watch_service, test_config):
|
||||
"""Test handling new file creation."""
|
||||
project_dir = test_config.home
|
||||
|
||||
table = watch_service.generate_table()
|
||||
assert table is not None
|
||||
# Setup changes
|
||||
new_file = project_dir / "new_note.md"
|
||||
changes = {(Change.added, str(new_file))}
|
||||
|
||||
# Create the file
|
||||
content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# New Note
|
||||
Test content
|
||||
"""
|
||||
await create_test_file(new_file, content)
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify
|
||||
entity = await watch_service.sync_service.entity_repository.get_by_file_path("new_note.md")
|
||||
assert entity is not None
|
||||
assert entity.title == "new_note.md"
|
||||
|
||||
# Check event was recorded
|
||||
events = [e for e in watch_service.state.recent_events if e.action == "new"]
|
||||
assert len(events) == 1
|
||||
assert events[0].path == "new_note.md"
|
||||
assert events[0].status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_changes(watch_service, mock_sync_service):
|
||||
"""Test handling file changes."""
|
||||
await watch_service.handle_changes(watch_service.config.home)
|
||||
async def test_handle_file_modify(watch_service, test_config):
|
||||
"""Test handling file modifications."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Check sync service was called
|
||||
mock_sync_service.sync.assert_called_once_with(watch_service.config.home)
|
||||
# Create initial file
|
||||
test_file = project_dir / "test_note.md"
|
||||
initial_content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Test Note
|
||||
Initial content
|
||||
"""
|
||||
await create_test_file(test_file, initial_content)
|
||||
|
||||
# Check events were recorded
|
||||
# Initial sync
|
||||
await watch_service.sync_service.sync(project_dir)
|
||||
|
||||
# Modify file
|
||||
modified_content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Test Note
|
||||
Modified content
|
||||
"""
|
||||
await create_test_file(test_file, modified_content)
|
||||
|
||||
# Setup changes
|
||||
changes = {(Change.modified, str(test_file))}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify
|
||||
entity = await watch_service.sync_service.entity_repository.get_by_file_path("test_note.md")
|
||||
assert entity is not None
|
||||
|
||||
# Check event was recorded
|
||||
events = [e for e in watch_service.state.recent_events if e.action == "modified"]
|
||||
assert len(events) == 1
|
||||
assert events[0].path == "test_note.md"
|
||||
assert events[0].status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_file_delete(watch_service, test_config):
|
||||
"""Test handling file deletion."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Create initial file
|
||||
test_file = project_dir / "to_delete.md"
|
||||
content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Delete Test
|
||||
Test content
|
||||
"""
|
||||
await create_test_file(test_file, content)
|
||||
|
||||
# Initial sync
|
||||
await watch_service.sync_service.sync(project_dir)
|
||||
|
||||
# Delete file
|
||||
test_file.unlink()
|
||||
|
||||
# Setup changes
|
||||
changes = {(Change.deleted, str(test_file))}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify
|
||||
entity = await watch_service.sync_service.entity_repository.get_by_file_path("to_delete.md")
|
||||
assert entity is None
|
||||
|
||||
# Check event was recorded
|
||||
events = [e for e in watch_service.state.recent_events if e.action == "deleted"]
|
||||
assert len(events) == 1
|
||||
assert events[0].path == "to_delete.md"
|
||||
assert events[0].status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_file_move(watch_service, test_config):
|
||||
"""Test handling file moves."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Create initial file
|
||||
old_path = project_dir / "old" / "test_move.md"
|
||||
content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Move Test
|
||||
Test content
|
||||
"""
|
||||
await create_test_file(old_path, content)
|
||||
|
||||
# Initial sync
|
||||
await watch_service.sync_service.sync(project_dir)
|
||||
initial_entity = await watch_service.sync_service.entity_repository.get_by_file_path(
|
||||
"old/test_move.md"
|
||||
)
|
||||
|
||||
# Move file
|
||||
new_path = project_dir / "new" / "moved_file.md"
|
||||
new_path.parent.mkdir(parents=True)
|
||||
old_path.rename(new_path)
|
||||
|
||||
# Setup changes
|
||||
changes = {(Change.deleted, str(old_path)), (Change.added, str(new_path))}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify
|
||||
moved_entity = await watch_service.sync_service.entity_repository.get_by_file_path(
|
||||
"new/moved_file.md"
|
||||
)
|
||||
assert moved_entity is not None
|
||||
assert moved_entity.id == initial_entity.id # Same entity, new path
|
||||
|
||||
# Original path should no longer exist
|
||||
old_entity = await watch_service.sync_service.entity_repository.get_by_file_path(
|
||||
"old/test_move.md"
|
||||
)
|
||||
assert old_entity is None
|
||||
|
||||
# Check event was recorded
|
||||
events = [e for e in watch_service.state.recent_events if e.action == "moved"]
|
||||
assert len(events) == 1
|
||||
assert events[0].path == "old/test_move.md -> new/moved_file.md"
|
||||
assert events[0].status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_concurrent_changes(watch_service, test_config):
|
||||
"""Test handling multiple file changes happening close together."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Create multiple files with small delays to simulate concurrent changes
|
||||
async def create_files():
|
||||
# Create first file
|
||||
file1 = project_dir / "note1.md"
|
||||
await create_test_file(file1, "First note")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Create second file
|
||||
file2 = project_dir / "note2.md"
|
||||
await create_test_file(file2, "Second note")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Modify first file
|
||||
await create_test_file(file1, "Modified first note")
|
||||
|
||||
return file1, file2
|
||||
|
||||
# Create files and collect changes
|
||||
file1, file2 = await create_files()
|
||||
|
||||
# Setup combined changes
|
||||
changes = {
|
||||
(Change.added, str(file1)),
|
||||
(Change.modified, str(file1)),
|
||||
(Change.added, str(file2)),
|
||||
}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify both files were processed
|
||||
entity1 = await watch_service.sync_service.entity_repository.get_by_file_path("note1.md")
|
||||
entity2 = await watch_service.sync_service.entity_repository.get_by_file_path("note2.md")
|
||||
|
||||
assert entity1 is not None
|
||||
assert entity2 is not None
|
||||
|
||||
# Check events were recorded in correct order
|
||||
events = watch_service.state.recent_events
|
||||
assert len(events) == 4 # new, modified, moved, deleted
|
||||
|
||||
# Check specific events
|
||||
actions = [e.action for e in events]
|
||||
assert "new" in actions
|
||||
assert "modified" in actions
|
||||
assert "moved" in actions
|
||||
assert "deleted" in actions
|
||||
assert "modified" not in actions # only process file once
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_rapid_move(watch_service, test_config):
|
||||
"""Test handling rapid move operations."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Create initial file
|
||||
original_path = project_dir / "original.md"
|
||||
content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Move Test
|
||||
Test content for rapid moves
|
||||
"""
|
||||
await create_test_file(original_path, content)
|
||||
await watch_service.sync_service.sync(project_dir)
|
||||
|
||||
# Perform rapid moves
|
||||
temp_path = project_dir / "temp.md"
|
||||
final_path = project_dir / "final.md"
|
||||
|
||||
original_path.rename(temp_path)
|
||||
await asyncio.sleep(0.1)
|
||||
temp_path.rename(final_path)
|
||||
|
||||
# Setup changes that might come in various orders
|
||||
changes = {
|
||||
(Change.deleted, str(original_path)),
|
||||
(Change.added, str(temp_path)),
|
||||
(Change.deleted, str(temp_path)),
|
||||
(Change.added, str(final_path)),
|
||||
}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify final state
|
||||
final_entity = await watch_service.sync_service.entity_repository.get_by_file_path("final.md")
|
||||
assert final_entity is not None
|
||||
|
||||
# Intermediate paths should not exist
|
||||
original_entity = await watch_service.sync_service.entity_repository.get_by_file_path(
|
||||
"original.md"
|
||||
)
|
||||
temp_entity = await watch_service.sync_service.entity_repository.get_by_file_path("temp.md")
|
||||
assert original_entity is None
|
||||
assert temp_entity is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_delete_then_add(watch_service, test_config):
|
||||
"""Test handling rapid move operations."""
|
||||
project_dir = test_config.home
|
||||
|
||||
# Create initial file
|
||||
original_path = project_dir / "original.md"
|
||||
content = """---
|
||||
type: knowledge
|
||||
---
|
||||
# Move Test
|
||||
Test content for rapid moves
|
||||
"""
|
||||
await create_test_file(original_path, content)
|
||||
|
||||
# Setup changes that might come in various orders
|
||||
changes = {
|
||||
(Change.deleted, str(original_path)),
|
||||
(Change.added, str(original_path)),
|
||||
}
|
||||
|
||||
# Handle changes
|
||||
await watch_service.handle_changes(project_dir, changes)
|
||||
|
||||
# Verify final state
|
||||
original_entity = await watch_service.sync_service.entity_repository.get_by_file_path(
|
||||
"original.md"
|
||||
)
|
||||
assert original_entity is None # delete event is handled
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Test edge cases in the WatchService."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from watchfiles import Change
|
||||
|
||||
|
||||
def test_filter_changes_valid_path(watch_service, test_config):
|
||||
"""Test the filter_changes method with valid non-hidden paths."""
|
||||
# Regular file path
|
||||
assert (
|
||||
watch_service.filter_changes(Change.added, str(test_config.home / "valid_file.txt")) is True
|
||||
)
|
||||
|
||||
# Nested path
|
||||
assert (
|
||||
watch_service.filter_changes(
|
||||
Change.added, str(test_config.home / "nested" / "valid_file.txt")
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_filter_changes_hidden_path(watch_service, test_config):
|
||||
"""Test the filter_changes method with hidden files/directories."""
|
||||
# Hidden file (starts with dot)
|
||||
assert (
|
||||
watch_service.filter_changes(Change.added, str(test_config.home / ".hidden_file.txt"))
|
||||
is False
|
||||
)
|
||||
|
||||
# File in hidden directory
|
||||
assert (
|
||||
watch_service.filter_changes(
|
||||
Change.added, str(test_config.home / ".hidden_dir" / "file.txt")
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
# Deeply nested hidden directory
|
||||
assert (
|
||||
watch_service.filter_changes(
|
||||
Change.added, str(test_config.home / "valid" / ".hidden" / "file.txt")
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_filter_changes_invalid_path(watch_service, test_config):
|
||||
"""Test the filter_changes method with invalid paths."""
|
||||
# Path outside of config.home
|
||||
outside_path = Path("/tmp/outside_path.txt")
|
||||
assert watch_service.filter_changes(Change.added, str(outside_path)) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_changes_empty_set(watch_service, test_config):
|
||||
"""Test handle_changes with an empty set (no processed files)."""
|
||||
# Mock write_status to avoid file operations
|
||||
with patch.object(watch_service, "write_status", return_value=None):
|
||||
# Capture console output to verify
|
||||
with patch.object(watch_service.console, "print") as mock_print:
|
||||
# Call handle_changes with empty set
|
||||
await watch_service.handle_changes(test_config.home, set())
|
||||
|
||||
# Verify divider wasn't printed (processed is empty)
|
||||
mock_print.assert_not_called()
|
||||
|
||||
# Verify last_scan was updated
|
||||
assert watch_service.state.last_scan is not None
|
||||
|
||||
# Verify synced_files wasn't changed
|
||||
assert watch_service.state.synced_files == 0
|
||||
Reference in New Issue
Block a user