diff --git a/src/basic_memory/repository/search_repository.py b/src/basic_memory/repository/search_repository.py index 727d62bf..36e989bc 100644 --- a/src/basic_memory/repository/search_repository.py +++ b/src/basic_memory/repository/search_repository.py @@ -1,6 +1,7 @@ """Repository for search operations.""" import json +import re import time from dataclasses import dataclass from datetime import datetime @@ -120,23 +121,113 @@ class SearchRepository: logger.error(f"Error initializing search index: {e}") raise e - def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: - """Prepare a search term for FTS5 query. - + def _prepare_boolean_query(self, query: str) -> str: + """Prepare a Boolean query by quoting individual terms while preserving operators. + Args: - term: The search term to prepare - is_prefix: Whether to add prefix search capability (* suffix) - - For FTS5: - - Boolean operators (AND, OR, NOT) are preserved for complex queries - - Terms with FTS5 special characters are quoted to prevent syntax errors - - Simple terms get prefix wildcards for better matching + query: A Boolean query like "tier1-test AND unicode" or "(hello OR world) NOT test" + + Returns: + A properly formatted Boolean query with quoted terms that need quoting """ - # Check for explicit boolean operators - if present, return the term as is - boolean_operators = [" AND ", " OR ", " NOT "] - if any(op in f" {term} " for op in boolean_operators): + # Define Boolean operators and their boundaries + boolean_pattern = r'(\bAND\b|\bOR\b|\bNOT\b)' + + # Split the query by Boolean operators, keeping the operators + parts = re.split(boolean_pattern, query) + + processed_parts = [] + for part in parts: + part = part.strip() + if not part: + continue + + # If it's a Boolean operator, keep it as is + if part in ['AND', 'OR', 'NOT']: + processed_parts.append(part) + else: + # Handle parentheses specially - they should be preserved for grouping + if '(' in part or ')' in part: + # Parse parenthetical expressions carefully + processed_part = self._prepare_parenthetical_term(part) + processed_parts.append(processed_part) + else: + # This is a search term - for Boolean queries, don't add prefix wildcards + prepared_term = self._prepare_single_term(part, is_prefix=False) + processed_parts.append(prepared_term) + + return " ".join(processed_parts) + + def _prepare_parenthetical_term(self, term: str) -> str: + """Prepare a term that contains parentheses, preserving the parentheses for grouping. + + Args: + term: A term that may contain parentheses like "(hello" or "world)" or "(hello OR world)" + + Returns: + A properly formatted term with parentheses preserved + """ + # Handle terms that start/end with parentheses but may contain quotable content + result = "" + i = 0 + while i < len(term): + if term[i] in '()': + # Preserve parentheses as-is + result += term[i] + i += 1 + else: + # Find the next parenthesis or end of string + start = i + while i < len(term) and term[i] not in '()': + i += 1 + + # Extract the content between parentheses + content = term[start:i].strip() + if content: + # Only quote if it actually needs quoting (has hyphens, special chars, etc) + # but don't quote if it's just simple words + if self._needs_quoting(content): + escaped_content = content.replace('"', '""') + result += f'"{escaped_content}"' + else: + result += content + + return result + + def _needs_quoting(self, term: str) -> bool: + """Check if a term needs to be quoted for FTS5 safety. + + Args: + term: The term to check + + Returns: + True if the term should be quoted + """ + if not term or not term.strip(): + return False + + # Characters that indicate we should quote (excluding parentheses which are valid syntax) + needs_quoting_chars = [" ", ".", ":", ";", ",", "<", ">", "?", "/", "-", "'", '"', + "[", "]", "{", "}", "+", "!", "@", "#", "$", "%", "^", "&", + "=", "|", "\\", "~", "`"] + + return any(c in term for c in needs_quoting_chars) + + def _prepare_single_term(self, term: str, is_prefix: bool = True) -> str: + """Prepare a single search term (no Boolean operators). + + Args: + term: A single search term + is_prefix: Whether to add prefix search capability (* suffix) + + Returns: + A properly formatted single term + """ + if not term or not term.strip(): return term - + + term = term.strip() + # Check if term is already a proper wildcard pattern (alphanumeric + *) # e.g., "hello*", "test*world" - these should be left alone if "*" in term and all(c.isalnum() or c in "*_-" for c in term): @@ -218,6 +309,26 @@ class SearchRepository: return term + def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str: + """Prepare a search term for FTS5 query. + + Args: + term: The search term to prepare + is_prefix: Whether to add prefix search capability (* suffix) + + For FTS5: + - Boolean operators (AND, OR, NOT) are preserved for complex queries + - Terms with FTS5 special characters are quoted to prevent syntax errors + - Simple terms get prefix wildcards for better matching + """ + # Check for explicit boolean operators - if present, process as Boolean query + boolean_operators = [" AND ", " OR ", " NOT "] + if any(op in f" {term} " for op in boolean_operators): + return self._prepare_boolean_query(term) + + # For non-Boolean queries, use the single term preparation logic + return self._prepare_single_term(term, is_prefix) + async def search( self, search_text: Optional[str] = None, @@ -242,19 +353,10 @@ class SearchRepository: # For wildcard searches, don't add any text conditions - return all results pass else: - # Check for explicit boolean operators - only detect them in proper boolean contexts - has_boolean = any(op in f" {search_text} " for op in [" AND ", " OR ", " NOT "]) - - if has_boolean: - # If boolean operators are present, use the raw query - # No need to prepare it, FTS5 will understand the operators - params["text"] = search_text - conditions.append("(title MATCH :text OR content_stems MATCH :text)") - else: - # Standard search with term preparation - processed_text = self._prepare_search_term(search_text.strip()) - params["text"] = processed_text - conditions.append("(title MATCH :text OR content_stems MATCH :text)") + # Use _prepare_search_term to handle both Boolean and non-Boolean queries + processed_text = self._prepare_search_term(search_text.strip()) + params["text"] = processed_text + conditions.append("(title MATCH :text OR content_stems MATCH :text)") # Handle title match search if title: diff --git a/tests/repository/test_search_repository.py b/tests/repository/test_search_repository.py index 8261a52a..33ea6046 100644 --- a/tests/repository/test_search_repository.py +++ b/tests/repository/test_search_repository.py @@ -329,6 +329,21 @@ class TestSearchTermPreparation: == "(hello AND world) OR test" ) + def test_hyphenated_terms_with_boolean_operators(self, search_repository): + """Hyphenated terms with Boolean operators should be properly quoted.""" + # Test the specific case from the GitHub issue + result = search_repository._prepare_search_term("tier1-test AND unicode") + assert result == '"tier1-test" AND unicode' + + # Test other hyphenated Boolean combinations + assert search_repository._prepare_search_term("multi-word OR single") == '"multi-word" OR single' + assert search_repository._prepare_search_term("well-formed NOT badly-formed") == '"well-formed" NOT "badly-formed"' + assert search_repository._prepare_search_term("test-case AND (hello OR world)") == '"test-case" AND (hello OR world)' + + # Test mixed special characters with Boolean operators + assert search_repository._prepare_search_term("config.json AND test-file") == '"config.json" AND "test-file"' + assert search_repository._prepare_search_term("C++ OR python-script") == '"C++" OR "python-script"' + def test_programming_terms_should_work(self, search_repository): """Programming-related terms with special chars should be searchable.""" # These should be quoted to handle special characters safely