fix project info stats tests

Signed-off-by: phernandez <paul@basicmachines.co>
This commit is contained in:
phernandez
2025-06-05 15:51:11 -05:00
parent 69d7610d47
commit 117fa44ecf
9 changed files with 302 additions and 70 deletions
@@ -2,12 +2,11 @@
from typing import Annotated, Optional
from dateparser import parse
from fastapi import APIRouter, Query
from loguru import logger
from basic_memory.deps import ContextServiceDep, EntityRepositoryDep
from basic_memory.schemas.base import TimeFrame
from basic_memory.schemas.base import TimeFrame, parse_timeframe
from basic_memory.schemas.memory import (
GraphContext,
normalize_memory_url,
@@ -40,7 +39,7 @@ async def recent(
f"Getting recent context: `{types}` depth: `{depth}` timeframe: `{timeframe}` page: `{page}` page_size: `{page_size}` max_related: `{max_related}`"
)
# Parse timeframe
since = parse(timeframe)
since = parse_timeframe(timeframe)
limit = page_size
offset = (page - 1) * page_size
@@ -78,7 +77,7 @@ async def get_memory_context(
memory_url = normalize_memory_url(uri)
# Parse timeframe
since = parse(timeframe) if timeframe else None
since = parse_timeframe(timeframe) if timeframe else None
limit = page_size
offset = (page - 1) * page_size
@@ -3,7 +3,7 @@
from fastapi import APIRouter, HTTPException, Path, Body
from typing import Optional
from basic_memory.deps import ProjectServiceDep
from basic_memory.deps import ProjectServiceDep, ProjectPathDep
from basic_memory.schemas import ProjectInfoResponse
from basic_memory.schemas.project_info import (
ProjectList,
@@ -22,9 +22,10 @@ project_resource_router = APIRouter(prefix="/projects", tags=["project_managemen
@project_router.get("/info", response_model=ProjectInfoResponse)
async def get_project_info(
project_service: ProjectServiceDep,
project: ProjectPathDep,
) -> ProjectInfoResponse:
"""Get comprehensive information about the current Basic Memory project."""
return await project_service.get_project_info()
"""Get comprehensive information about the specified Basic Memory project."""
return await project_service.get_project_info(project)
# Update a project
@@ -47,7 +48,7 @@ async def update_project(
"""
try: # pragma: no cover
# Get original project info for the response
old_project = ProjectItem(
old_project_info = ProjectItem(
name=project_name,
path=project_service.projects.get(project_name, ""),
)
@@ -61,7 +62,7 @@ async def update_project(
message=f"Project '{project_name}' updated successfully",
status="success",
default=(project_name == project_service.default_project),
old_project=old_project,
old_project=old_project_info,
new_project=ProjectItem(name=project_name, path=updated_path),
)
except ValueError as e: # pragma: no cover
@@ -5,12 +5,12 @@ It centralizes all prompt formatting logic that was previously in the MCP prompt
"""
from datetime import datetime, timezone
from dateparser import parse
from fastapi import APIRouter, HTTPException, status
from loguru import logger
from basic_memory.api.routers.utils import to_graph_context, to_search_results
from basic_memory.api.template_loader import template_loader
from basic_memory.schemas.base import parse_timeframe
from basic_memory.deps import (
ContextServiceDep,
EntityRepositoryDep,
@@ -51,7 +51,7 @@ async def continue_conversation(
f"Generating continue conversation prompt, topic: {request.topic}, timeframe: {request.timeframe}"
)
since = parse(request.timeframe) if request.timeframe else None
since = parse_timeframe(request.timeframe) if request.timeframe else None
# Initialize search results
search_results = []
@@ -96,7 +96,11 @@ async def switch_project(project_name: str, ctx: Context | None = None) -> str:
# Get project info to show summary
try:
response = await call_get(client, f"{project_config.project_url}/project/info")
response = await call_get(
client,
f"{project_config.project_url}/project/info",
params={"project_name": project_name},
)
project_info = ProjectInfoResponse.model_validate(response.json())
result = f"✓ Switched to {project_name} project\n\n"
@@ -163,7 +167,11 @@ async def get_current_project(ctx: Context | None = None) -> str:
result = f"Current project: {current_project}\n\n"
# get project stats
response = await call_get(client, f"{project_config.project_url}/project/info")
response = await call_get(
client,
f"{project_config.project_url}/project/info",
params={"project_name": current_project},
)
project_info = ProjectInfoResponse.model_validate(response.json())
result += f"{project_info.statistics.total_entities} entities\n"
+33 -5
View File
@@ -13,7 +13,7 @@ Key Concepts:
import mimetypes
import re
from datetime import datetime
from datetime import datetime, time
from pathlib import Path
from typing import List, Optional, Annotated, Dict
@@ -46,15 +46,43 @@ def to_snake_case(name: str) -> str:
return s2.lower()
def parse_timeframe(timeframe: str) -> datetime:
"""Parse timeframe with special handling for 'today' and other natural language expressions.
Args:
timeframe: Natural language timeframe like 'today', '1d', '1 week ago', etc.
Returns:
datetime: The parsed datetime for the start of the timeframe
Examples:
parse_timeframe('today') -> 2025-06-05 00:00:00 (start of today)
parse_timeframe('1d') -> 2025-06-04 14:50:00 (24 hours ago)
parse_timeframe('1 week ago') -> 2025-05-29 14:50:00 (1 week ago)
"""
if timeframe.lower() == "today":
# Return start of today (00:00:00)
return datetime.combine(datetime.now().date(), time.min)
else:
# Use dateparser for other formats
parsed = parse(timeframe)
if not parsed:
raise ValueError(f"Could not parse timeframe: {timeframe}")
return parsed
def validate_timeframe(timeframe: str) -> str:
"""Convert human readable timeframes to a duration relative to the current time."""
if not isinstance(timeframe, str):
raise ValueError("Timeframe must be a string")
# Parse relative time expression
parsed = parse(timeframe)
if not parsed:
raise ValueError(f"Could not parse timeframe: {timeframe}")
# Preserve special timeframe strings that need custom handling
special_timeframes = ["today"]
if timeframe.lower() in special_timeframes:
return timeframe.lower()
# Parse relative time expression using our enhanced parser
parsed = parse_timeframe(timeframe)
# Convert to duration
now = datetime.now()
+93 -45
View File
@@ -311,8 +311,11 @@ class ProjectService:
f"Changed default project to '{new_default.name}' as '{name}' was deactivated"
)
async def get_project_info(self) -> ProjectInfoResponse:
"""Get comprehensive information about the current Basic Memory project.
async def get_project_info(self, project_name: Optional[str] = None) -> ProjectInfoResponse:
"""Get comprehensive information about the specified Basic Memory project.
Args:
project_name: Name of the project to get info for. If None, uses the current config project.
Returns:
Comprehensive project information and statistics
@@ -320,19 +323,27 @@ class ProjectService:
if not self.repository: # pragma: no cover
raise ValueError("Repository is required for get_project_info")
# Get statistics
statistics = await self.get_statistics()
# Use specified project or fall back to config project
project_name = project_name or config.project
# Get project path from configuration
project_path = config_manager.projects.get(project_name)
if not project_path: # pragma: no cover
raise ValueError(f"Project '{project_name}' not found in configuration")
# Get activity metrics
activity = await self.get_activity_metrics()
# Get project from database to get project_id
db_project = await self.repository.get_by_name(project_name)
if not db_project: # pragma: no cover
raise ValueError(f"Project '{project_name}' not found in database")
# Get statistics for the specified project
statistics = await self.get_statistics(db_project.id)
# Get activity metrics for the specified project
activity = await self.get_activity_metrics(db_project.id)
# Get system status
system = self.get_system_status()
# Get current project information from config
project_name = config.project
project_path = str(config.home)
# Get enhanced project information from database
db_projects = await self.repository.get_active_projects()
db_projects_by_name = {p.name: p for p in db_projects}
@@ -363,60 +374,85 @@ class ProjectService:
system=system,
)
async def get_statistics(self) -> ProjectStatistics:
"""Get statistics about the current project."""
async def get_statistics(self, project_id: int) -> ProjectStatistics:
"""Get statistics about the specified project.
Args:
project_id: ID of the project to get statistics for (required).
"""
if not self.repository: # pragma: no cover
raise ValueError("Repository is required for get_statistics")
# Get basic counts
entity_count_result = await self.repository.execute_query(
text("SELECT COUNT(*) FROM entity")
text("SELECT COUNT(*) FROM entity WHERE project_id = :project_id"),
{"project_id": project_id},
)
total_entities = entity_count_result.scalar() or 0
observation_count_result = await self.repository.execute_query(
text("SELECT COUNT(*) FROM observation")
text(
"SELECT COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id"
),
{"project_id": project_id},
)
total_observations = observation_count_result.scalar() or 0
relation_count_result = await self.repository.execute_query(
text("SELECT COUNT(*) FROM relation")
text(
"SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id"
),
{"project_id": project_id},
)
total_relations = relation_count_result.scalar() or 0
unresolved_count_result = await self.repository.execute_query(
text("SELECT COUNT(*) FROM relation WHERE to_id IS NULL")
text(
"SELECT COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE r.to_id IS NULL AND e.project_id = :project_id"
),
{"project_id": project_id},
)
total_unresolved = unresolved_count_result.scalar() or 0
# Get entity counts by type
entity_types_result = await self.repository.execute_query(
text("SELECT entity_type, COUNT(*) FROM entity GROUP BY entity_type")
text(
"SELECT entity_type, COUNT(*) FROM entity WHERE project_id = :project_id GROUP BY entity_type"
),
{"project_id": project_id},
)
entity_types = {row[0]: row[1] for row in entity_types_result.fetchall()}
# Get observation counts by category
category_result = await self.repository.execute_query(
text("SELECT category, COUNT(*) FROM observation GROUP BY category")
text(
"SELECT o.category, COUNT(*) FROM observation o JOIN entity e ON o.entity_id = e.id WHERE e.project_id = :project_id GROUP BY o.category"
),
{"project_id": project_id},
)
observation_categories = {row[0]: row[1] for row in category_result.fetchall()}
# Get relation counts by type
relation_types_result = await self.repository.execute_query(
text("SELECT relation_type, COUNT(*) FROM relation GROUP BY relation_type")
text(
"SELECT r.relation_type, COUNT(*) FROM relation r JOIN entity e ON r.from_id = e.id WHERE e.project_id = :project_id GROUP BY r.relation_type"
),
{"project_id": project_id},
)
relation_types = {row[0]: row[1] for row in relation_types_result.fetchall()}
# Find most connected entities (most outgoing relations)
# Find most connected entities (most outgoing relations) - project filtered
connected_result = await self.repository.execute_query(
text("""
SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count, file_path
SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count, e.file_path
FROM entity e
JOIN relation r ON e.id = r.from_id
WHERE e.project_id = :project_id
GROUP BY e.id
ORDER BY relation_count DESC
LIMIT 10
""")
"""),
{"project_id": project_id},
)
most_connected = [
{
@@ -429,15 +465,16 @@ class ProjectService:
for row in connected_result.fetchall()
]
# Count isolated entities (no relations)
# Count isolated entities (no relations) - project filtered
isolated_result = await self.repository.execute_query(
text("""
SELECT COUNT(e.id)
FROM entity e
LEFT JOIN relation r1 ON e.id = r1.from_id
LEFT JOIN relation r2 ON e.id = r2.to_id
WHERE r1.id IS NULL AND r2.id IS NULL
""")
WHERE e.project_id = :project_id AND r1.id IS NULL AND r2.id IS NULL
"""),
{"project_id": project_id},
)
isolated_count = isolated_result.scalar() or 0
@@ -453,19 +490,25 @@ class ProjectService:
isolated_entities=isolated_count,
)
async def get_activity_metrics(self) -> ActivityMetrics:
"""Get activity metrics for the current project."""
async def get_activity_metrics(self, project_id: int) -> ActivityMetrics:
"""Get activity metrics for the specified project.
Args:
project_id: ID of the project to get activity metrics for (required).
"""
if not self.repository: # pragma: no cover
raise ValueError("Repository is required for get_activity_metrics")
# Get recently created entities
# Get recently created entities (project filtered)
created_result = await self.repository.execute_query(
text("""
SELECT id, title, permalink, entity_type, created_at, file_path
FROM entity
WHERE project_id = :project_id
ORDER BY created_at DESC
LIMIT 10
""")
"""),
{"project_id": project_id},
)
recently_created = [
{
@@ -479,14 +522,16 @@ class ProjectService:
for row in created_result.fetchall()
]
# Get recently updated entities
# Get recently updated entities (project filtered)
updated_result = await self.repository.execute_query(
text("""
SELECT id, title, permalink, entity_type, updated_at, file_path
FROM entity
WHERE project_id = :project_id
ORDER BY updated_at DESC
LIMIT 10
""")
"""),
{"project_id": project_id},
)
recently_updated = [
{
@@ -507,47 +552,50 @@ class ProjectService:
now.year - (1 if now.month <= 6 else 0), ((now.month - 6) % 12) or 12, 1
)
# Query for monthly entity creation
# Query for monthly entity creation (project filtered)
entity_growth_result = await self.repository.execute_query(
text(f"""
text("""
SELECT
strftime('%Y-%m', created_at) AS month,
COUNT(*) AS count
FROM entity
WHERE created_at >= '{six_months_ago.isoformat()}'
WHERE created_at >= :six_months_ago AND project_id = :project_id
GROUP BY month
ORDER BY month
""")
"""),
{"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
)
entity_growth = {row[0]: row[1] for row in entity_growth_result.fetchall()}
# Query for monthly observation creation
# Query for monthly observation creation (project filtered)
observation_growth_result = await self.repository.execute_query(
text(f"""
text("""
SELECT
strftime('%Y-%m', created_at) AS month,
strftime('%Y-%m', entity.created_at) AS month,
COUNT(*) AS count
FROM observation
INNER JOIN entity ON observation.entity_id = entity.id
WHERE entity.created_at >= '{six_months_ago.isoformat()}'
WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
GROUP BY month
ORDER BY month
""")
"""),
{"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
)
observation_growth = {row[0]: row[1] for row in observation_growth_result.fetchall()}
# Query for monthly relation creation
# Query for monthly relation creation (project filtered)
relation_growth_result = await self.repository.execute_query(
text(f"""
text("""
SELECT
strftime('%Y-%m', created_at) AS month,
strftime('%Y-%m', entity.created_at) AS month,
COUNT(*) AS count
FROM relation
INNER JOIN entity ON relation.from_id = entity.id
WHERE entity.created_at >= '{six_months_ago.isoformat()}'
WHERE entity.created_at >= :six_months_ago AND entity.project_id = :project_id
GROUP BY month
ORDER BY month
""")
"""),
{"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
)
relation_growth = {row[0]: row[1] for row in relation_growth_result.fetchall()}
+149 -1
View File
@@ -1,6 +1,7 @@
"""Tests for Pydantic schema validation and conversion."""
import pytest
from datetime import datetime, time, timedelta
from pydantic import ValidationError, BaseModel
from basic_memory.schemas import (
@@ -12,7 +13,7 @@ from basic_memory.schemas import (
RelationResponse,
)
from basic_memory.schemas.request import EditEntityRequest
from basic_memory.schemas.base import to_snake_case, TimeFrame
from basic_memory.schemas.base import to_snake_case, TimeFrame, parse_timeframe, validate_timeframe
def test_entity_project_name():
@@ -277,3 +278,150 @@ def test_edit_entity_request_replace_section_empty_section():
"section": "", # Empty string triggers validation
}
)
# New tests for timeframe parsing functions
class TestTimeframeParsing:
"""Test cases for parse_timeframe() and validate_timeframe() functions."""
def test_parse_timeframe_today(self):
"""Test that parse_timeframe('today') returns start of current day."""
result = parse_timeframe("today")
expected = datetime.combine(datetime.now().date(), time.min)
assert result == expected
assert result.hour == 0
assert result.minute == 0
assert result.second == 0
assert result.microsecond == 0
def test_parse_timeframe_today_case_insensitive(self):
"""Test that parse_timeframe handles 'today' case-insensitively."""
test_cases = ["today", "TODAY", "Today", "ToDay"]
expected = datetime.combine(datetime.now().date(), time.min)
for case in test_cases:
result = parse_timeframe(case)
assert result == expected
def test_parse_timeframe_other_formats(self):
"""Test that parse_timeframe works with other dateparser formats."""
now = datetime.now()
# Test 1d ago - should be approximately 24 hours ago
result_1d = parse_timeframe("1d")
expected_1d = now - timedelta(days=1)
diff = abs((result_1d - expected_1d).total_seconds())
assert diff < 60 # Within 1 minute tolerance
# Test yesterday - should be yesterday at same time
result_yesterday = parse_timeframe("yesterday")
# dateparser returns yesterday at current time, not start of yesterday
assert result_yesterday.date() == (now.date() - timedelta(days=1))
# Test 1 week ago
result_week = parse_timeframe("1 week ago")
expected_week = now - timedelta(weeks=1)
diff = abs((result_week - expected_week).total_seconds())
assert diff < 3600 # Within 1 hour tolerance
def test_parse_timeframe_invalid(self):
"""Test that parse_timeframe raises ValueError for invalid input."""
with pytest.raises(ValueError, match="Could not parse timeframe: invalid-timeframe"):
parse_timeframe("invalid-timeframe")
with pytest.raises(ValueError, match="Could not parse timeframe: not-a-date"):
parse_timeframe("not-a-date")
def test_validate_timeframe_preserves_special_cases(self):
"""Test that validate_timeframe preserves special timeframe strings."""
# Should preserve 'today' as-is
result = validate_timeframe("today")
assert result == "today"
# Should preserve case-normalized version
result = validate_timeframe("TODAY")
assert result == "today"
result = validate_timeframe("Today")
assert result == "today"
def test_validate_timeframe_converts_regular_formats(self):
"""Test that validate_timeframe converts regular formats to duration."""
# Test 1d format (should return as-is since it's already in standard format)
result = validate_timeframe("1d")
assert result == "1d"
# Test other formats get converted to days
result = validate_timeframe("yesterday")
assert result == "1d" # Yesterday is 1 day ago
# Test week format
result = validate_timeframe("1 week ago")
assert result == "7d" # 1 week = 7 days
def test_validate_timeframe_error_cases(self):
"""Test that validate_timeframe raises appropriate errors."""
# Invalid type
with pytest.raises(ValueError, match="Timeframe must be a string"):
validate_timeframe(123) # type: ignore
# Future timeframe
with pytest.raises(ValueError, match="Timeframe cannot be in the future"):
validate_timeframe("tomorrow")
# Too far in past (>365 days)
with pytest.raises(ValueError, match="Timeframe should be <= 1 year"):
validate_timeframe("2 years ago")
# Invalid format that can't be parsed
with pytest.raises(ValueError, match="Could not parse timeframe"):
validate_timeframe("not-a-real-timeframe")
def test_timeframe_annotation_with_today(self):
"""Test that TimeFrame annotation works correctly with 'today'."""
class TestModel(BaseModel):
timeframe: TimeFrame
# Should preserve 'today'
model = TestModel(timeframe="today")
assert model.timeframe == "today"
# Should work with other formats
model = TestModel(timeframe="1d")
assert model.timeframe == "1d"
model = TestModel(timeframe="yesterday")
assert model.timeframe == "1d"
def test_timeframe_integration_today_vs_1d(self):
"""Test the specific bug fix: 'today' vs '1d' behavior."""
class TestModel(BaseModel):
timeframe: TimeFrame
# 'today' should be preserved
today_model = TestModel(timeframe="today")
assert today_model.timeframe == "today"
# '1d' should also be preserved (it's already in standard format)
oneday_model = TestModel(timeframe="1d")
assert oneday_model.timeframe == "1d"
# When parsed by parse_timeframe, they should be different
today_parsed = parse_timeframe("today")
oneday_parsed = parse_timeframe("1d")
# 'today' should be start of today (00:00:00)
assert today_parsed.hour == 0
assert today_parsed.minute == 0
# '1d' should be 24 hours ago (same time yesterday)
now = datetime.now()
expected_1d = now - timedelta(days=1)
diff = abs((oneday_parsed - expected_1d).total_seconds())
assert diff < 60 # Within 1 minute
# They should be different times
assert today_parsed != oneday_parsed
+6 -6
View File
@@ -123,10 +123,10 @@ async def test_get_system_status(project_service: ProjectService):
@pytest.mark.asyncio
async def test_get_statistics(project_service: ProjectService, test_graph):
async def test_get_statistics(project_service: ProjectService, test_graph, test_project):
"""Test getting statistics."""
# Get statistics
statistics = await project_service.get_statistics()
statistics = await project_service.get_statistics(test_project.id)
# Assert it returns a valid ProjectStatistics object
assert isinstance(statistics, ProjectStatistics)
@@ -135,10 +135,10 @@ async def test_get_statistics(project_service: ProjectService, test_graph):
@pytest.mark.asyncio
async def test_get_activity_metrics(project_service: ProjectService, test_graph):
async def test_get_activity_metrics(project_service: ProjectService, test_graph, test_project):
"""Test getting activity metrics."""
# Get activity metrics
metrics = await project_service.get_activity_metrics()
metrics = await project_service.get_activity_metrics(test_project.id)
# Assert it returns a valid ActivityMetrics object
assert isinstance(metrics, ActivityMetrics)
@@ -147,10 +147,10 @@ async def test_get_activity_metrics(project_service: ProjectService, test_graph)
@pytest.mark.asyncio
async def test_get_project_info(project_service: ProjectService, test_graph):
async def test_get_project_info(project_service: ProjectService, test_graph, test_project):
"""Test getting full project info."""
# Get project info
info = await project_service.get_project_info()
info = await project_service.get_project_info(test_project.name)
# Assert it returns a valid ProjectInfoResponse object
assert isinstance(info, ProjectInfoResponse)