From 16869867da9ad686fe9ebdb598252eb2e9046765 Mon Sep 17 00:00:00 2001 From: Drew Cain Date: Sat, 13 Jun 2026 14:17:55 -0500 Subject: [PATCH] fix(core): gate hybrid FTS relaxation with the service's eligibility rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Codex review on #994: the hybrid FTS branch opted into OR-relaxation for every query shape, but SearchService's relaxed FTS path deliberately rejects short queries and numeric identifiers because OR-relaxing them over-broadens — and in hybrid the relaxed FTS-only rows normalize to 1.0 and can outrank the vector result the user wanted (e.g. "SPEC 16", "root note 1", "New Feature"). relaxed_query_words now enforces the same eligibility as SearchService._is_relaxed_fts_fallback_eligible: tokenize on [A-Za-z0-9]+ and return None when there are fewer than three tokens or any token is a pure digit (in addition to the existing quoted/boolean guards). Both the SQLite and Postgres relaxed retries route through this helper, so the hybrid path now relaxes exactly the query shapes the service does. Tests updated for the tightened eligibility (short + numeric queries no longer relax); SQLite + Postgres relaxation suites green, full repository/search-service/link-resolver suite green (442 passed), ty clean. Signed-off-by: Drew Cain --- .../repository/search_repository_base.py | 18 ++++++++++++++++-- tests/repository/test_search_repository.py | 19 +++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/basic_memory/repository/search_repository_base.py b/src/basic_memory/repository/search_repository_base.py index 52e2d0a8..06fd4304 100644 --- a/src/basic_memory/repository/search_repository_base.py +++ b/src/basic_memory/repository/search_repository_base.py @@ -53,14 +53,28 @@ RELAXATION_STOPWORDS = frozenset( def relaxed_query_words(search_text: Optional[str]) -> Optional[list[str]]: """Content-bearing words for OR-relaxing a strict full-text query. - Returns None when relaxation must not apply: empty input, quoted phrases, - or explicit boolean queries (user intent is not second-guessed). + Returns None when relaxation must not apply. These eligibility rules match + SearchService._is_relaxed_fts_fallback_eligible so the hybrid FTS branch + relaxes exactly the same query shapes as the service-level FTS path: + + - empty / quoted / explicit-boolean queries (user intent is not + second-guessed); + - fewer than three alphanumeric tokens (short queries like "New Feature" + over-broaden under OR — and in hybrid the relaxed FTS-only rows normalize + to 1.0 and can outrank the vector result the user wanted); + - any pure-digit token ("root note 1", "SPEC 16") — identifier-like queries + over-broaden and create false positives under OR. """ if not search_text: return None stripped = search_text.strip() if '"' in stripped or any(op in f" {stripped} " for op in (" AND ", " OR ", " NOT ")): return None + # Eligibility checks run on raw alphanumeric tokens (parity with the + # service), before stopword filtering. + tokens = re.findall(r"[A-Za-z0-9]+", stripped.lower()) + if len(tokens) < 3 or any(token.isdigit() for token in tokens): + return None words = [word.strip("?!.,;:") for word in stripped.split()] words = [ word diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 0068b3f1..3a95f1da 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -1156,16 +1156,19 @@ async def test_relaxed_query_drops_stopwords(search_repository): @pytest.mark.asyncio async def test_relaxed_query_respects_user_intent(search_repository): - # Explicit boolean and quoted queries are not second-guessed (both backends). - if is_postgres_backend(search_repository): - relaxer = search_repository._relaxed_tsquery_text - single = "single:*" - else: - relaxer = search_repository._relaxed_fts_text - single = "single*" + # Eligibility matches the service-level relaxation (both backends): quoted, + # boolean, short (<3 tokens), and numeric-identifier queries are not relaxed. + relaxer = ( + search_repository._relaxed_tsquery_text + if is_postgres_backend(search_repository) + else search_repository._relaxed_fts_text + ) assert relaxer("alpha AND beta") is None assert relaxer('"exact phrase"') is None - assert relaxer("single") == single + assert relaxer("single") is None # < 3 tokens + assert relaxer("New Feature") is None # < 3 tokens (link title) + assert relaxer("root note 1") is None # numeric identifier token + assert relaxer("SPEC 16 design") is None # numeric token assert relaxer(None) is None