diff --git a/src/basic_memory/schemas/activity.py b/src/basic_memory/schemas/activity.py index 89349161..57228545 100644 --- a/src/basic_memory/schemas/activity.py +++ b/src/basic_memory/schemas/activity.py @@ -43,7 +43,6 @@ class TimeFrame: class ActivityType(str, Enum): """Types of activities that can be tracked.""" - DOCUMENT = "document" ENTITY = "entity" RELATION = "relation" @@ -67,7 +66,6 @@ class ActivityChange(BaseModel): class ActivitySummary(BaseModel): """Summary statistics about recent activity.""" - document_changes: int = Field(default=0, description="Number of document changes") entity_changes: int = Field(default=0, description="Number of entity changes") relation_changes: int = Field(default=0, description="Number of relation changes") most_active_paths: List[str] = Field( diff --git a/src/basic_memory/services/activity_service.py b/src/basic_memory/services/activity_service.py index f5c9ccee..82f75029 100644 --- a/src/basic_memory/services/activity_service.py +++ b/src/basic_memory/services/activity_service.py @@ -3,8 +3,7 @@ from datetime import datetime, timezone from typing import List, Optional, Sequence -from . import EntityService, DocumentService, RelationService -from ..models import Document +from . import EntityService, RelationService from ..schemas.activity import ( ActivityChange, ActivitySummary, @@ -21,12 +20,10 @@ class ActivityService: def __init__( self, entity_service: EntityService, - document_service: DocumentService, relation_service: RelationService, ): """Initialize with required services.""" self.entity_service = entity_service - self.document_service = document_service self.relation_service = relation_service async def get_recent_activity( @@ -58,8 +55,6 @@ class ActivityService: for activity_type in types_to_fetch: if activity_type == ActivityType.ENTITY: changes.extend(await self._get_entity_changes(since)) - elif activity_type == ActivityType.DOCUMENT: - changes.extend(await self._get_document_changes(since)) elif activity_type == ActivityType.RELATION: changes.extend(await self._get_relation_changes(since)) @@ -71,7 +66,6 @@ class ActivityService: # Generate summary summary = ActivitySummary( - document_changes=len([c for c in changes if c.activity_type == ActivityType.DOCUMENT]), entity_changes=len([c for c in changes if c.activity_type == ActivityType.ENTITY]), relation_changes=len([c for c in changes if c.activity_type == ActivityType.RELATION]), most_active_paths=self._get_most_active_paths(changes) @@ -109,31 +103,6 @@ class ActivityService: return changes - async def _get_document_changes(self, since: datetime) -> List[ActivityChange]: - """Get recent document changes.""" - # Query documents updated since the cutoff - documents: Sequence[Document] = await self.document_service.get_modified_since(since) - - changes = [] - for doc in documents: - # Ensure timestamps are timezone-aware - created_at = doc.created_at.replace(tzinfo=timezone.utc) if doc.created_at.tzinfo is None else doc.created_at - updated_at = doc.updated_at.replace(tzinfo=timezone.utc) if doc.updated_at.tzinfo is None else doc.updated_at - - change_type = ChangeType.CREATED if created_at >= since else ChangeType.UPDATED - - changes.append( - ActivityChange( - activity_type=ActivityType.DOCUMENT, - change_type=change_type, - timestamp=updated_at, - path_id=doc.path_id, - summary=f"{change_type.value.title()} document: {doc.path_id}", - content=None # Document content lives in the filesystem - ) - ) - - return changes async def _get_relation_changes(self, since: datetime) -> List[ActivityChange]: """Get recent relation changes.""" diff --git a/src/basic_memory/services/search_service.py b/src/basic_memory/services/search_service.py index 17243ff9..b02f3459 100644 --- a/src/basic_memory/services/search_service.py +++ b/src/basic_memory/services/search_service.py @@ -5,7 +5,7 @@ from typing import List, Optional, Any from fastapi import BackgroundTasks from loguru import logger -from basic_memory.models import Document, Entity +from basic_memory.models import Entity from basic_memory.repository.search_repository import SearchRepository from basic_memory.services.entity_service import EntityService from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType diff --git a/src/basic_memory/sync/file_change_scanner.py b/src/basic_memory/sync/file_change_scanner.py index f66bae16..c32fc829 100644 --- a/src/basic_memory/sync/file_change_scanner.py +++ b/src/basic_memory/sync/file_change_scanner.py @@ -5,7 +5,7 @@ from typing import Dict, Sequence from loguru import logger -from basic_memory.models import Document, Entity +from basic_memory.models import Entity from basic_memory.repository.entity_repository import EntityRepository from basic_memory.sync.utils import DbState, SyncReport, ScanResult from basic_memory.utils.file_utils import compute_checksum @@ -129,7 +129,7 @@ class FileChangeScanner: return report async def get_db_file_paths( - self, db_records: Sequence[Document | Entity] + self, db_records: Sequence[Entity] ) -> Dict[str, DbState]: """Get file_path and checksums from database. Args: diff --git a/tests/api/test_activity_router.py b/tests/api/test_activity_router.py index 6840c604..c3508d8f 100644 --- a/tests/api/test_activity_router.py +++ b/tests/api/test_activity_router.py @@ -29,7 +29,7 @@ async def test_get_recent_activity_with_filters(client: AsyncClient): "/activity/recent", params={ "timeframe": "1h", - "activity_types": [ActivityType.DOCUMENT.value], + "activity_types": [ActivityType.ENTITY.value], "include_content": False } ) @@ -41,5 +41,5 @@ async def test_get_recent_activity_with_filters(client: AsyncClient): # Verify all changes are document type for change in data["changes"]: - assert change["activity_type"] == ActivityType.DOCUMENT.value + assert change["activity_type"] == ActivityType.ENTITY.value assert change["content"] is None # Content excluded \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index af293025..0cb70781 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,7 +46,6 @@ def test_config(tmp_path) -> ProjectConfig: ) config.home = tmp_path - (tmp_path / config.documents_dir.name).mkdir(parents=True, exist_ok=True) (tmp_path / config.knowledge_dir.name).mkdir(parents=True, exist_ok=True) return config diff --git a/tests/schemas/test_activity_schemas.py b/tests/schemas/test_activity_schemas.py index 7a5a0eb1..ad429ced 100644 --- a/tests/schemas/test_activity_schemas.py +++ b/tests/schemas/test_activity_schemas.py @@ -47,7 +47,7 @@ def test_activity_change_model(): """Test ActivityChange model.""" now = datetime.utcnow() change = ActivityChange( - activity_type=ActivityType.DOCUMENT, + activity_type=ActivityType.ENTITY, change_type=ChangeType.CREATED, timestamp=now, path_id="test/path", @@ -55,7 +55,7 @@ def test_activity_change_model(): content="Test content" ) - assert change.activity_type == ActivityType.DOCUMENT + assert change.activity_type == ActivityType.ENTITY assert change.change_type == ChangeType.CREATED assert change.timestamp == now assert change.path_id == "test/path" @@ -66,13 +66,11 @@ def test_activity_change_model(): def test_activity_summary_model(): """Test ActivitySummary model.""" summary = ActivitySummary( - document_changes=5, entity_changes=3, relation_changes=2, most_active_paths=["path1", "path2"] ) - assert summary.document_changes == 5 assert summary.entity_changes == 3 assert summary.relation_changes == 2 assert summary.most_active_paths == ["path1", "path2"] @@ -82,7 +80,7 @@ def test_recent_activity_model(): """Test RecentActivity model.""" now = datetime.utcnow() change = ActivityChange( - activity_type=ActivityType.DOCUMENT, + activity_type=ActivityType.ENTITY, change_type=ChangeType.CREATED, timestamp=now, path_id="test/path", @@ -90,7 +88,7 @@ def test_recent_activity_model(): ) summary = ActivitySummary( - document_changes=1 + entity_changes=1 ) activity = RecentActivity( @@ -102,4 +100,4 @@ def test_recent_activity_model(): assert activity.timeframe == "1d" assert len(activity.changes) == 1 assert activity.changes[0].path_id == "test/path" - assert activity.summary.document_changes == 1 \ No newline at end of file + assert activity.summary.entity_changes == 1 \ No newline at end of file diff --git a/tests/services/test_search_service.py b/tests/services/test_search_service.py index 2664a02d..f43ea5c5 100644 --- a/tests/services/test_search_service.py +++ b/tests/services/test_search_service.py @@ -121,40 +121,6 @@ async def test_search_date_filter(search_service, test_entity): assert len(results) == 0 -@pytest.mark.asyncio -async def test_index_document(search_service, test_document): - """Test indexing a document""" - content = """# Test Document - -This is a test document with some searchable content. -It contains technical information about implementation.""" - - await search_service.index_document(test_document, content) - - # Search for document content - results = await search_service.search(SearchQuery(text="searchable content")) - assert len(results) == 1 - assert results[0].path_id == test_document.path_id - assert results[0].type == SearchItemType.DOCUMENT - - # Verify metadata - assert results[0].metadata["title"] == "Test Document" - assert results[0].metadata["type"] == "technical" - - -@pytest.mark.asyncio -async def test_update_document_index(search_service, test_document): - """Test updating an indexed document""" - # Initial indexing - await search_service.index_document(test_document, "Initial content") - - # Update with new content - await search_service.index_document(test_document, "Updated content with new terms") - - # Search for new terms - results = await search_service.search(SearchQuery(text="new terms")) - assert len(results) == 1 - @pytest.mark.asyncio async def test_reindex_all( diff --git a/tests/sync/test_file_change_scanner.py b/tests/sync/test_file_change_scanner.py index 20fde9bb..51dbfd25 100644 --- a/tests/sync/test_file_change_scanner.py +++ b/tests/sync/test_file_change_scanner.py @@ -4,8 +4,7 @@ from pathlib import Path import pytest -from basic_memory.models import Document, Entity -from basic_memory.repository import DocumentRepository +from basic_memory.models import Entity from basic_memory.sync import FileChangeScanner from basic_memory.sync.utils import DbState from basic_memory.utils.file_utils import compute_checksum @@ -67,7 +66,7 @@ async def test_scan_with_unreadable_file(file_change_scanner: FileChangeScanner, @pytest.mark.asyncio async def test_detect_new_files( - file_change_scanner: FileChangeScanner, temp_dir: Path, document_repository: DocumentRepository + file_change_scanner: FileChangeScanner, temp_dir: Path, ): """Test detection of new files.""" # Create new file @@ -116,17 +115,6 @@ async def test_detect_deleted_files(file_change_scanner: FileChangeScanner, temp assert path in changes.deleted -@pytest.mark.asyncio -async def test_get_db_state_documents(file_change_scanner: FileChangeScanner): - """Test converting document records to file states.""" - doc = Document(path_id="test.md", file_path="test.md", checksum="test-checksum") - - db_records = await file_change_scanner.get_db_file_paths([doc]) - - assert len(db_records) == 1 - assert "test.md" in db_records - assert db_records["test.md"].checksum == "test-checksum" - @pytest.mark.asyncio async def test_get_db_state_entities(file_change_scanner: FileChangeScanner): @@ -140,15 +128,6 @@ async def test_get_db_state_entities(file_change_scanner: FileChangeScanner): assert db_records["concept/test.md"].checksum == "test-checksum" -@pytest.mark.asyncio -async def test_get_db_state_does_not_skip_missing_checksum(file_change_scanner: FileChangeScanner): - """Test that get_db_state skips records with missing checksums.""" - doc = Document(path_id="test.md", file_path="test.md", checksum=None) - - db_records = await file_change_scanner.get_db_file_paths([doc]) - - assert len(db_records) == 1 - @pytest.mark.asyncio async def test_empty_directory(file_change_scanner: FileChangeScanner, temp_dir: Path): diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index b64daa1a..4392f26a 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -24,7 +24,6 @@ async def test_sync_empty_directories(sync_service: SyncService, test_config: Pr await sync_service.sync(test_config) # Should not raise exceptions for empty dirs - assert (test_config.documents_dir).exists() assert (test_config.knowledge_dir).exists() @@ -34,8 +33,19 @@ async def test_sync_file_modified_during_sync( ): """Test handling of files that change during sync process.""" # Create initial files - doc_path = test_config.documents_dir / "changing.md" - await create_test_file(doc_path, "Initial content") + doc_path = test_config.knowledge_dir / "changing.md" + await create_test_file(doc_path, """ +--- +type: knowledge +id: changing +created: 2024-01-01 +modified: 2024-01-01 +--- +# Knowledge File + +## Observations +- This is a test +""") # Setup async modification during sync async def modify_file(): @@ -46,7 +56,7 @@ async def test_sync_file_modified_during_sync( await asyncio.gather(sync_service.sync(test_config), modify_file()) # Verify final state - doc = await sync_service.document_service.repository.find_by_path_id("changing.md") + doc = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("changing") assert doc is not None # File should have a checksum, even if it's from either version assert doc.checksum is not None @@ -90,120 +100,4 @@ modified: 2024-01-01 assert updated.checksum is not None -@pytest.mark.asyncio -async def test_sync_mixed_document_types(sync_service: SyncService, test_config: ProjectConfig): - """Test handling documents and knowledge files with similar paths.""" - # Create a document - doc_content = "# Regular Document" - await create_test_file(test_config.documents_dir / "test.md", doc_content) - # Create a knowledge file - knowledge_content = """ ---- -type: knowledge -id: concept/test -created: 2024-01-01 -modified: 2024-01-01 ---- -# Knowledge File - -## Observations -- This is a test -""" - await create_test_file(test_config.knowledge_dir / "concept/test.md", knowledge_content) - - # Run sync - await sync_service.sync(test_config) - - # Verify both types exist correctly - doc = await sync_service.document_service.repository.find_by_path_id("test.md") - assert doc is not None - - entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id("concept/test") - assert entity is not None - assert len(entity.observations) == 1 - - -@pytest.mark.asyncio -async def test_sync_performance_large_files(sync_service: SyncService, test_config: ProjectConfig): - """Test sync performance with larger files.""" - # Create a large document with many lines - large_doc = ["Line " + str(i) for i in range(1000)] - await create_test_file(test_config.documents_dir / "large.md", "\n".join(large_doc)) - - # Create a knowledge file with many observations - observations = [f"- Observation {i}" for i in range(100)] - knowledge_content = f""" ---- -type: knowledge -id: concept/large -created: 2024-01-01 -modified: 2024-01-01 ---- -# Large Entity - -## Observations -{chr(10).join(observations)} -""" - await create_test_file(test_config.knowledge_dir / "concept/large.md", knowledge_content) - - # Time the sync - start_time = asyncio.get_event_loop().time() - await sync_service.sync(test_config) - duration = asyncio.get_event_loop().time() - start_time - - # Verify everything synced - doc = await sync_service.document_service.repository.find_by_path_id("large.md") - assert doc is not None - - entity = await sync_service.knowledge_sync_service.entity_service.get_by_path_id( - "concept/large" - ) - assert entity is not None - assert len(entity.observations) == 100 - - # Basic performance check - should sync in reasonable time - assert duration < 5 # Should complete in under 5 seconds - - -# skip for now - until we handle concurrency with db - -# @pytest.mark.asyncio -# async def test_sync_concurrent_updates( -# sync_service: SyncService, -# test_config: ProjectConfig -# ): -# """Test concurrent syncs maintain database consistency.""" -# doc1_path = test_config.documents_dir / "doc1.md" -# doc2_path = test_config.documents_dir / "doc2.md" -# -# await create_test_file(doc1_path, "Doc 1 content") -# await create_test_file(doc2_path, "Doc 2 content") -# -# # Run concurrent syncs -# results = await asyncio.gather( -# sync_service.sync(test_config), -# sync_service.sync(test_config), -# return_exceptions=True -# ) -# -# # Check no exceptions were raised -# for r in results: -# if isinstance(r, Exception): -# print(r) -# assert not isinstance(r, Exception) -# -# # Verify database consistency -# docs = await sync_service.document_service.repository.find_all() -# assert len(docs) == 2 # No duplicates -# assert {d.path_id for d in docs} == {"doc1.md", "doc2.md"} -# -# # Both files should have valid checksums -# for doc in docs: -# assert doc.checksum is not None -# -# # Running another sync should not change anything -# await sync_service.sync(test_config) -# docs_after = await sync_service.document_service.repository.find_all() -# assert len(docs_after) == 2 -# assert {d.path_id for d in docs_after} == {"doc1.md", "doc2.md"}