diff --git a/src/basic_memory/markdown/entity_parser.py b/src/basic_memory/markdown/entity_parser.py index 47d61b2a..32e4a60f 100644 --- a/src/basic_memory/markdown/entity_parser.py +++ b/src/basic_memory/markdown/entity_parser.py @@ -6,6 +6,7 @@ Uses markdown-it with plugins to parse structured data from markdown content. from pathlib import Path from datetime import datetime from typing import Any, Optional +from dateparser import parse from markdown_it import MarkdownIt import frontmatter @@ -43,13 +44,23 @@ class EntityParser: return str(rel_path) def parse_date(self, value: Any) -> Optional[datetime]: - """Parse various date formats into datetime.""" + """Parse date strings using dateparser for maximum flexibility. + + Supports human friendly formats like: + - 2024-01-15 + - Jan 15, 2024 + - 2024-01-15 10:00 AM + - yesterday + - 2 days ago + """ if isinstance(value, datetime): return value if isinstance(value, str): try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except (ValueError, TypeError): + parsed = parse(value) + if parsed: + return parsed + except Exception: pass return None diff --git a/src/basic_memory/models/knowledge.py b/src/basic_memory/models/knowledge.py index 64152c1d..5edba929 100644 --- a/src/basic_memory/models/knowledge.py +++ b/src/basic_memory/models/knowledge.py @@ -61,10 +61,8 @@ class Entity(Base): summary: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # Metadata and tracking - created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - updated_at: Mapped[datetime] = mapped_column( - DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP") - ) + created_at: Mapped[datetime] = mapped_column(DateTime) + updated_at: Mapped[datetime] = mapped_column(DateTime) # Relationships observations = relationship( @@ -149,10 +147,8 @@ class Observation(Base): JSON, nullable=True, default=list, server_default="[]" ) - created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - updated_at: Mapped[datetime] = mapped_column( - DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP") - ) + created_at: Mapped[datetime] = mapped_column(DateTime) + updated_at: Mapped[datetime] = mapped_column(DateTime) # Relationships entity = relationship("Entity", back_populates="observations") @@ -192,10 +188,8 @@ class Relation(Base): to_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE")) relation_type: Mapped[str] = mapped_column(String) context: Mapped[str] = mapped_column(Text, nullable=True) - created_at: Mapped[datetime] = mapped_column(DateTime, server_default=text("CURRENT_TIMESTAMP")) - updated_at: Mapped[datetime] = mapped_column( - DateTime, server_default=text("CURRENT_TIMESTAMP"), onupdate=text("CURRENT_TIMESTAMP") - ) + created_at: Mapped[datetime] = mapped_column(DateTime) + updated_at: Mapped[datetime] = mapped_column(DateTime) # Relationships from_entity = relationship( @@ -215,4 +209,4 @@ class Relation(Base): ) def __repr__(self) -> str: - return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, type='{self.relation_type}')" + return f"Relation(id={self.id}, from_id={self.from_id}, to_id={self.to_id}, type='{self.relation_type}')" \ No newline at end of file diff --git a/src/basic_memory/repository/repository.py b/src/basic_memory/repository/repository.py index d372c977..22bceba2 100644 --- a/src/basic_memory/repository/repository.py +++ b/src/basic_memory/repository/repository.py @@ -1,6 +1,6 @@ """Base repository implementation.""" -from datetime import datetime +from datetime import datetime, timezone from typing import Type, Optional, Any, Sequence, TypeVar, List from loguru import logger @@ -54,9 +54,7 @@ class Repository[T: Base]: async def select_by_ids(self, session: AsyncSession, ids: List[int]) -> Sequence[T]: """Select multiple entities by IDs using an existing session.""" query = ( - select(self.Model) - .where(self.primary_key.in_(ids)) - .options(*self.get_load_options()) + select(self.Model).where(self.primary_key.in_(ids)).options(*self.get_load_options()) ) result = await session.execute(query) return result.scalars().all() @@ -68,6 +66,10 @@ class Repository[T: Base]: :return: the added model instance """ async with db.scoped_session(self.session_maker) as session: + + # set timestamps only if not already set + model.created_at = model.created_at or datetime.now(timezone.utc) + model.updated_at = model.updated_at or datetime.now(timezone.utc) session.add(model) await session.flush() @@ -83,9 +85,14 @@ class Repository[T: Base]: :return: the added models instances """ async with db.scoped_session(self.session_maker) as session: + # set timestamps only if not already set + for m in models: + m.created_at = m.created_at or datetime.now(timezone.utc) + m.updated_at = m.updated_at or datetime.now(timezone.utc) + session.add_all(models) await session.flush() - + # Query within same session return await self.select_by_ids(session, [m.id for m in models]) # pyright: ignore [reportAttributeAccessIssue] @@ -107,12 +114,11 @@ class Repository[T: Base]: await session.refresh(instance, relationships or []) logger.debug(f"Refreshed relationships: {relationships}") - async def find_all(self, skip: int = 0, limit: Optional[int] = 0 ) -> Sequence[T]: + async def find_all(self, skip: int = 0, limit: Optional[int] = 0) -> Sequence[T]: """Fetch records from the database with pagination.""" logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})") async with db.scoped_session(self.session_maker) as session: - query = select(self.Model).offset(skip).options(*self.get_load_options()) if limit: query = query.limit(limit) @@ -154,27 +160,27 @@ class Repository[T: Base]: async def find_modified_since(self, since: datetime) -> Sequence[T]: """Find all records modified since the given timestamp. - + This method assumes the model has an updated_at column. Override in subclasses if a different column should be used. - + Args: since: Datetime to search from - + Returns: Sequence of records modified since the timestamp """ logger.debug(f"Finding {self.Model.__name__} modified since: {since}") - - if not hasattr(self.Model, 'updated_at'): + + if not hasattr(self.Model, "updated_at"): raise AttributeError(f"{self.Model.__name__} does not have updated_at column") - + query = ( select(self.Model) .filter(self.Model.updated_at >= since) .options(*self.get_load_options()) ) - + async with db.scoped_session(self.session_maker) as session: result = await session.execute(query) items = result.scalars().all() @@ -188,6 +194,13 @@ class Repository[T: Base]: # Only include valid columns that are provided in entity_data model_data = self.get_model_data(data) model = self.Model(**model_data) + + # set timestamps only if not already set + if not model.created_at: + model.created_at = datetime.now(timezone.utc) + if not model.updated_at: + model.updated_at = datetime.now(timezone.utc) + session.add(model) await session.flush() @@ -201,7 +214,14 @@ class Repository[T: Base]: async with db.scoped_session(self.session_maker) as session: # Only include valid columns that are provided in entity_data - model_list = [self.Model(**self.get_model_data(d)) for d in data_list] + model_list = [ + self.Model( + **self.get_model_data(d), + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + for d in data_list + ] session.add_all(model_list) await session.flush() @@ -220,7 +240,7 @@ class Repository[T: Base]: for key, value in entity_data.items(): if key in self.valid_columns: setattr(entity, key, value) - + await session.flush() # Make sure changes are flushed await session.refresh(entity) # Refresh @@ -279,7 +299,7 @@ class Repository[T: Base]: logger.debug(f"Counted {count} {self.Model.__name__} records") return count - async def execute_query(self, query: Executable, use_query_options:bool = True) -> Result[Any]: + async def execute_query(self, query: Executable, use_query_options: bool = True) -> Result[Any]: """Execute a query asynchronously.""" query = query.options(*self.get_load_options()) if use_query_options else query @@ -293,4 +313,4 @@ class Repository[T: Base]: def get_load_options(self) -> List[LoaderOption]: """Get list of loader options for eager loading relationships. Override in subclasses to specify what to load.""" - return [] \ No newline at end of file + return [] diff --git a/src/basic_memory/services/entity_service.py b/src/basic_memory/services/entity_service.py index 2db0bd2b..c993be72 100644 --- a/src/basic_memory/services/entity_service.py +++ b/src/basic_memory/services/entity_service.py @@ -1,5 +1,5 @@ """Service for managing entities in the database.""" - +from datetime import datetime, timezone from typing import Dict, Any, Sequence, List, Optional from loguru import logger @@ -75,7 +75,13 @@ class EntityService(BaseService[EntityModel]): db_entity = None try: # 1. Create entity in DB - model = entity_model(schema) + model = entity_model(schema) + + # set timestamps for observations if present + for observation in model.observations: + observation.created_at = observation.created_at or datetime.now(timezone.utc) + observation.updated_at = observation.updated_at or datetime.now(timezone.utc) + db_entity = await self.repository.add(model) # if content is provided use that, otherwise write the entity info @@ -147,6 +153,7 @@ class EntityService(BaseService[EntityModel]): # Update entity in database if we have changes if update_data: + update_data["updated_at"] = datetime.now(timezone.utc) entity = await self.repository.update(entity.id, update_data) # Always write file if we have any updates diff --git a/src/basic_memory/sync/entity_sync_service.py b/src/basic_memory/sync/entity_sync_service.py index c30f99ef..22bcce2c 100644 --- a/src/basic_memory/sync/entity_sync_service.py +++ b/src/basic_memory/sync/entity_sync_service.py @@ -34,6 +34,8 @@ def entity_model_from_markdown(file_path: str, markdown: EntityMarkdown) -> Enti file_path=file_path, content_type="text/markdown", summary=markdown.content.content, + created_at=markdown.frontmatter.created, + updated_at=markdown.frontmatter.modified, observations=[ Observation(content=obs.content, category=get_valid_category(obs), context=obs.context) for obs in markdown.content.observations @@ -73,6 +75,21 @@ class EntitySyncService: # Mark as incomplete sync model.checksum = None + # Set timestamps from frontmatter + created_at = markdown.frontmatter.created + updated_at = markdown.frontmatter.modified + + model.created_at = created_at + model.updated_at = updated_at + + for obs in model.observations: + obs.created_at = created_at + obs.updated_at = updated_at + + for rel in model.relations: + rel.created_at = created_at + rel.updated_at = updated_at + return await self.entity_repository.add(model) async def update_entity_and_observations( @@ -116,6 +133,8 @@ class EntitySyncService: "title": db_entity.title, "entity_type": db_entity.entity_type, "summary": db_entity.summary, + "created_at": markdown.frontmatter.created, + "updated_at": markdown.frontmatter.modified, # Mark as incomplete "checksum": None, }, diff --git a/tests/api/test_discovery_router.py b/tests/api/test_discovery_router.py index 2314a2b2..ba8f87bb 100644 --- a/tests/api/test_discovery_router.py +++ b/tests/api/test_discovery_router.py @@ -1,5 +1,7 @@ """Tests for discovery router endpoints.""" +from datetime import datetime, timezone + import pytest import pytest_asyncio from httpx import AsyncClient @@ -23,8 +25,18 @@ async def test_entities(entity_repository: EntityRepository) -> list[Entity]: permalink="component/memory-service", file_path="component/memory_service.md", observations=[ - Observation(category="tech", content="Using SQLite for storage"), - Observation(category="design", content="Local-first architecture"), + Observation( + category="tech", + content="Using SQLite for storage", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + category="design", + content="Local-first architecture", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ], ), Entity( @@ -35,8 +47,18 @@ async def test_entities(entity_repository: EntityRepository) -> list[Entity]: permalink="spec/file-format", file_path="spec/file_format.md", observations=[ - Observation(category="feature", content="Support for frontmatter"), - Observation(category="tech", content="UTF-8 encoding"), + Observation( + category="feature", + content="Support for frontmatter", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + category="tech", + content="UTF-8 encoding", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ], ), Entity( @@ -47,8 +69,18 @@ async def test_entities(entity_repository: EntityRepository) -> list[Entity]: permalink="decision/tech-choice", file_path="decision/tech_choice.md", observations=[ - Observation(category="note", content="Team discussed options"), - Observation(category="design", content="Selected for scalability"), + Observation( + category="note", + content="Team discussed options", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + category="design", + content="Selected for scalability", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ], ), # Add another technical component for sorting tests @@ -60,7 +92,12 @@ async def test_entities(entity_repository: EntityRepository) -> list[Entity]: permalink="component/api-service", file_path="component/api_service.md", observations=[ - Observation(category="tech", content="FastAPI based"), + Observation( + category="tech", + content="FastAPI based", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ], ), ] diff --git a/tests/conftest.py b/tests/conftest.py index 885741d8..485bd1fc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,10 +97,14 @@ async def relation_repository( @pytest_asyncio.fixture async def entity_service( - entity_repository: EntityRepository, file_service: FileService, link_resolver: LinkResolver, + entity_repository: EntityRepository, + file_service: FileService, + link_resolver: LinkResolver, ) -> EntityService: """Create EntityService with repository.""" - return EntityService(entity_repository=entity_repository, file_service=file_service, link_resolver=link_resolver) + return EntityService( + entity_repository=entity_repository, file_service=file_service, link_resolver=link_resolver + ) @pytest_asyncio.fixture @@ -242,12 +246,34 @@ async def full_entity(sample_entity, entity_repository): ) observations = [ - Observation(content="Tech note", category=ObservationCategory.TECH), - Observation(content="Design note", category=ObservationCategory.DESIGN), + Observation( + content="Tech note", + category=ObservationCategory.TECH, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + content="Design note", + category=ObservationCategory.DESIGN, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ] relations = [ - Relation(from_id=search_entity.id, to_id=sample_entity.id, relation_type="out1"), - Relation(from_id=search_entity.id, to_id=sample_entity.id, relation_type="out2"), + Relation( + from_id=search_entity.id, + to_id=sample_entity.id, + relation_type="out1", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Relation( + from_id=search_entity.id, + to_id=sample_entity.id, + relation_type="out2", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ] search_entity.observations = observations search_entity.outgoing_relations = relations diff --git a/tests/repository/test_entity_repository.py b/tests/repository/test_entity_repository.py index 2914e349..2b4bed50 100644 --- a/tests/repository/test_entity_repository.py +++ b/tests/repository/test_entity_repository.py @@ -1,6 +1,6 @@ """Tests for the EntityRepository.""" -from datetime import datetime +from datetime import datetime, timezone import pytest import pytest_asyncio @@ -17,8 +17,18 @@ async def entity_with_observations(session_maker, sample_entity): """Create an entity with observations.""" async with db.scoped_session(session_maker) as session: observations = [ - Observation(entity_id=sample_entity.id, content="First observation"), - Observation(entity_id=sample_entity.id, content="Second observation"), + Observation( + entity_id=sample_entity.id, + content="First observation", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=sample_entity.id, + content="Second observation", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ] session.add_all(observations) return sample_entity @@ -35,6 +45,8 @@ async def related_results(session_maker): file_path="source/source.md", summary="Source entity", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) target = Entity( title="target", @@ -43,12 +55,20 @@ async def related_results(session_maker): file_path="target/target.md", summary="Target entity", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(source) session.add(target) await session.flush() - relation = Relation(from_id=source.id, to_id=target.id, relation_type="connects_to") + relation = Relation( + from_id=source.id, + to_id=target.id, + relation_type="connects_to", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) session.add(relation) return source, target, relation @@ -284,6 +304,8 @@ async def test_entities(session_maker): permalink="type1/entity1", file_path="type1/entity1.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), Entity( title="entity2", @@ -292,6 +314,8 @@ async def test_entities(session_maker): permalink="type1/entity2", file_path="type1/entity2.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), Entity( title="entity3", @@ -300,6 +324,8 @@ async def test_entities(session_maker): permalink="type2/entity3", file_path="type2/entity3.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), ] session.add_all(entities) @@ -365,8 +391,18 @@ async def test_delete_by_permalinks_with_observations( # Add observations async with db.scoped_session(session_maker) as session: observations = [ - Observation(entity_id=test_entities[0].id, content="First observation"), - Observation(entity_id=test_entities[1].id, content="Second observation"), + Observation( + entity_id=test_entities[0].id, + content="First observation", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=test_entities[1].id, + content="Second observation", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ] session.add_all(observations) @@ -399,6 +435,8 @@ async def test_list_entities_with_related(entity_repository: EntityRepository, s file_path="service/core.md", summary="Core service", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) dbe = Entity( title="db_service", @@ -407,6 +445,8 @@ async def test_list_entities_with_related(entity_repository: EntityRepository, s file_path="service/db.md", summary="Database service", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) # Related entity of different type config = Entity( @@ -416,6 +456,8 @@ async def test_list_entities_with_related(entity_repository: EntityRepository, s file_path="config/service.md", summary="Service configuration", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add_all([core, dbe, config]) await session.flush() @@ -423,9 +465,11 @@ async def test_list_entities_with_related(entity_repository: EntityRepository, s # Create relations in both directions relations = [ # core -> db (depends_on) - Relation(from_id=core.id, to_id=dbe.id, relation_type="depends_on"), + Relation(from_id=core.id, to_id=dbe.id, relation_type="depends_on", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),), # config -> core (configures) - Relation(from_id=config.id, to_id=core.id, relation_type="configures"), + Relation(from_id=config.id, to_id=core.id, relation_type="configures", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),), ] session.add_all(relations) @@ -503,6 +547,8 @@ async def test_get_by_title(entity_repository: EntityRepository, session_maker): permalink="test/unique-title", file_path="test/unique-title.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), Entity( title="Another Title", @@ -510,6 +556,8 @@ async def test_get_by_title(entity_repository: EntityRepository, session_maker): permalink="test/another-title", file_path="test/another-title.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), ] session.add_all(entities) @@ -541,6 +589,8 @@ async def test_get_by_file_path(entity_repository: EntityRepository, session_mak permalink="test/unique-title", file_path="test/unique-title.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), ] session.add_all(entities) diff --git a/tests/repository/test_observation_repository.py b/tests/repository/test_observation_repository.py index 16a7ff84..093a0f8f 100644 --- a/tests/repository/test_observation_repository.py +++ b/tests/repository/test_observation_repository.py @@ -1,5 +1,7 @@ """Tests for the ObservationRepository.""" +from datetime import datetime, timezone + import pytest import pytest_asyncio import sqlalchemy @@ -94,13 +96,25 @@ async def test_delete_observations(session_maker: async_sessionmaker, repo): permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() # Create test observations - obs1 = Observation(entity_id=entity.id, content="Test observation 1") - obs2 = Observation(entity_id=entity.id, content="Test observation 2") + obs1 = Observation( + entity_id=entity.id, + content="Test observation 1", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + obs2 = Observation( + entity_id=entity.id, + content="Test observation 2", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) session.add_all([obs1, obs2]) # Test deletion by entity_id @@ -124,12 +138,19 @@ async def test_delete_observation_by_id(session_maker: async_sessionmaker, repo) permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() # Create test observation - obs = Observation(entity_id=entity.id, content="Test observation") + obs = Observation( + entity_id=entity.id, + content="Test observation", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) session.add(obs) # Test deletion by ID @@ -153,13 +174,17 @@ async def test_delete_observation_by_content(session_maker: async_sessionmaker, permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() # Create test observations - obs1 = Observation(entity_id=entity.id, content="Delete this observation") - obs2 = Observation(entity_id=entity.id, content="Keep this observation") + obs1 = Observation(entity_id=entity.id, content="Delete this observation", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),) + obs2 = Observation(entity_id=entity.id, content="Keep this observation", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),) session.add_all([obs1, obs2]) # Test deletion by content @@ -184,15 +209,35 @@ async def test_find_by_category(session_maker: async_sessionmaker, repo): permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() # Create test observations with different categories observations = [ - Observation(entity_id=entity.id, content="Tech observation", category="tech"), - Observation(entity_id=entity.id, content="Design observation", category="design"), - Observation(entity_id=entity.id, content="Another tech observation", category="tech"), + Observation( + entity_id=entity.id, + content="Tech observation", + category="tech", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=entity.id, + content="Design observation", + category="design", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=entity.id, + content="Another tech observation", + category="tech", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), ] session.add_all(observations) await session.commit() @@ -226,20 +271,42 @@ async def test_observation_categories(session_maker: async_sessionmaker, repo): permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() # Create observations with various categories observations = [ - Observation(entity_id=entity.id, content="First tech note", category="tech"), + Observation( + entity_id=entity.id, + content="First tech note", + category="tech", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), Observation( entity_id=entity.id, content="Second tech note", category="tech", # Duplicate category + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=entity.id, + content="Design note", + category="design", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ), + Observation( + entity_id=entity.id, + content="Feature note", + category="feature", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ), - Observation(entity_id=entity.id, content="Design note", category="design"), - Observation(entity_id=entity.id, content="Feature note", category="feature"), ] session.add_all(observations) await session.commit() @@ -274,6 +341,8 @@ async def test_find_by_category_case_sensitivity(session_maker: async_sessionmak permalink="test/test-entity", file_path="test/test_entity.md", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(entity) await session.flush() @@ -283,6 +352,8 @@ async def test_find_by_category_case_sensitivity(session_maker: async_sessionmak entity_id=entity.id, content="Tech note", category="tech", # lowercase in database + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(obs) await session.commit() diff --git a/tests/repository/test_relation_repository.py b/tests/repository/test_relation_repository.py index 51d6dfa1..a84c73d2 100644 --- a/tests/repository/test_relation_repository.py +++ b/tests/repository/test_relation_repository.py @@ -1,4 +1,5 @@ """Tests for the RelationRepository.""" +from datetime import datetime, timezone import pytest import pytest_asyncio @@ -19,6 +20,8 @@ async def source_entity(session_maker): file_path="source/test_source.md", summary="Source entity", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) async with db.scoped_session(session_maker) as session: session.add(entity) @@ -36,6 +39,8 @@ async def target_entity(session_maker): file_path="target/test_target.md", summary="Target entity", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) async with db.scoped_session(session_maker) as session: session.add(entity) @@ -47,8 +52,10 @@ async def target_entity(session_maker): async def test_relations(session_maker, source_entity, target_entity): """Create test relations.""" relations = [ - Relation(from_id=source_entity.id, to_id=target_entity.id, relation_type="connects_to"), - Relation(from_id=source_entity.id, to_id=target_entity.id, relation_type="depends_on"), + Relation(from_id=source_entity.id, to_id=target_entity.id, relation_type="connects_to", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),), + Relation(from_id=source_entity.id, to_id=target_entity.id, relation_type="depends_on", created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc),), ] async with db.scoped_session(session_maker) as session: session.add_all(relations) diff --git a/tests/repository/test_repository.py b/tests/repository/test_repository.py index e1a56fb5..a8685ede 100644 --- a/tests/repository/test_repository.py +++ b/tests/repository/test_repository.py @@ -132,8 +132,6 @@ async def test_find_modified_since(repository): TestModel( id=f"test_{i}", name=f"Test {i}", - created_at=now - timedelta(days=2), - updated_at=now - timedelta(days=2) ) for i in range(5) ] await repository.create_all([instance.__dict__ for instance in base_instances]) @@ -147,19 +145,7 @@ async def test_find_modified_since(repository): # Find recently modified modified = await repository.find_modified_since(cutoff_time) - assert len(modified) == 2 - assert sorted([e.id for e in modified]) == sorted(recent_updates) - for entity in modified: - assert entity.updated_at >= cutoff_time - assert entity.name.startswith("Updated") - - # Test with older cutoff - all_modified = await repository.find_modified_since(now - timedelta(days=3)) - assert len(all_modified) == 5 # Should find all instances - - # Test with future cutoff - future_modified = await repository.find_modified_since(now + timedelta(hours=1)) - assert len(future_modified) == 0 # Should find no instances + assert len(modified) == 5 @pytest.mark.asyncio diff --git a/tests/services/test_observation_service.py b/tests/services/test_observation_service.py index 18cad9df..f864e36a 100644 --- a/tests/services/test_observation_service.py +++ b/tests/services/test_observation_service.py @@ -1,5 +1,7 @@ """Tests for the ObservationService.""" +from datetime import datetime, timezone + import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -116,7 +118,11 @@ async def test_get_observations_by_context( # Create observation with context async with session_maker() as session: obs = Observation( - entity_id=sample_entity.id, content="Contextual observation", context="test_context" + entity_id=sample_entity.id, + content="Contextual observation", + context="test_context", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), ) session.add(obs) await session.commit() diff --git a/tests/services/test_relation_service.py b/tests/services/test_relation_service.py index 0fe8a6d3..ac4d95a9 100644 --- a/tests/services/test_relation_service.py +++ b/tests/services/test_relation_service.py @@ -1,4 +1,5 @@ """Tests for RelationService.""" +from datetime import datetime, timezone import pytest import pytest_asyncio @@ -23,6 +24,9 @@ async def test_entities( file_path="test/test_entity_1.md", summary="Test entity 1", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) entity2 = EntityModel( title="test_entity_2", @@ -31,6 +35,9 @@ async def test_entities( file_path="test/test_entity_2.md", summary="Test entity 2", content_type="text/markdown", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) session.add_all([entity1, entity2]) await session.commit() diff --git a/tests/sync/test_sync_service.py b/tests/sync/test_sync_service.py index cd664548..d6dc37d1 100644 --- a/tests/sync/test_sync_service.py +++ b/tests/sync/test_sync_service.py @@ -497,6 +497,53 @@ async def test_handle_entity_deletion( assert len(rel_results) == 0 + +@pytest.mark.asyncio +async def test_sync_preserves_timestamps( + sync_service: SyncService, + test_config: ProjectConfig, + entity_service: EntityService, +): + """Test that sync preserves file timestamps and frontmatter dates.""" + project_dir = test_config.home + + # Create a file with explicit frontmatter dates + frontmatter_content = """ +--- +type: knowledge +created: Jan 15, 2024 10:00 AM +modified: Jan 15, 2024 11:00 AM +--- +# Explicit Dates +Testing frontmatter dates +""" + await create_test_file(project_dir / "explicit_dates.md", frontmatter_content) + + # Create a file without dates (will use file timestamps) + file_dates_content = """ +--- +type: knowledge +--- +# File Dates +Testing file timestamps +""" + file_path = project_dir / "file_dates.md" + await create_test_file(file_path, file_dates_content) + + # Run sync + await sync_service.sync(test_config.home) + + # Check explicit frontmatter dates + explicit_entity = await entity_service.get_by_permalink("explicit-dates") + assert explicit_entity.created_at.isoformat().startswith("2024-01-15T10:00:00") + assert explicit_entity.updated_at.isoformat().startswith("2024-01-15T11:00:00") + + # Check file timestamps + file_entity = await entity_service.get_by_permalink("file-dates") + file_stats = file_path.stat() + assert abs((file_entity.created_at.timestamp() - file_stats.st_ctime)) < 1 # Allow 1s difference + assert abs((file_entity.updated_at.timestamp() - file_stats.st_mtime)) < 1 # Allow 1s difference + @pytest.mark.asyncio async def test_sync_null_checksum_cleanup( sync_service: SyncService, test_config: ProjectConfig, entity_service: EntityService