Fix solr text field config

WIP: the re-indexing after migration on start doesn't work anymore

Author: eikek

bases/renku_data_services/data_api/main.py

@@ -176,6 +176,7 @@ async def setup_rclone_validator(app: Sanic) -> None:
     @app.after_server_start
     async def ready(app: Sanic) -> None:
         """Application ready event handler."""
+        logger.info(f">>>>>> SHOULD RUN RE-INDEX NOW. {getattr(app.ctx, "solr_reindex", False)}")
         if getattr(app.ctx, "solr_reindex", False):
             logger.info("Creating solr reindex task, as required by migrations.")
             app.add_task(solr_reindex(dependency_manager.search_reprovisioning))

components/renku_data_services/search/solr_token.py

@@ -1,6 +1,5 @@
 """Model for creating solr lucene queries."""
 
-import re
 from collections.abc import Iterable
 from datetime import UTC, datetime
 from typing import NewType
@@ -186,9 +185,8 @@ def public_only() -> SolrToken:
 
 
 def content_all(text: str) -> SolrToken:
-    """Search the content_all field with fuzzy searching each term."""
-    terms: list[SolrToken] = list(map(lambda s: SolrToken(__escape_query(s)), re.split("\\s+", text)))
-    terms_str = "(" + " ".join(terms) + ")"
+    """Search the content_all field."""
+    terms_str = "(" + __escape_query(text) + "~)"
     return SolrToken(f"{Fields.content_all}:{terms_str}")
 
 

components/renku_data_services/solr/entity_schema.py

@@ -60,21 +60,21 @@ class Analyzers:
     """A collection of analyzers."""
 
     text_index: Final[Analyzer] = Analyzer(
-        tokenizer=Tokenizers.uax29UrlEmail,
+        tokenizer=Tokenizers.whitespace,
         filters=[
+            Filters.word,
+            Filters.flattenGraph,
             Filters.lowercase,
-            Filters.stop,
             Filters.english_minimal_stem,
             Filters.ascii_folding,
             Filters.edgeNgram(2, 8, True),
         ],
     )
 
     text_query: Final[Analyzer] = Analyzer(
-        tokenizer=Tokenizers.uax29UrlEmail,
+        tokenizer=Tokenizers.whitespace,
         filters=[
             Filters.lowercase,
-            Filters.stop,
             Filters.english_minimal_stem,
             Filters.ascii_folding,
         ],
@@ -189,4 +189,9 @@ class FieldTypes:
         ],
         requires_reindex=True,
     ),
+    SchemaMigration(
+        version=15,
+        commands=[ReplaceCommand(FieldTypes.text), ReplaceCommand(FieldTypes.text_all)],
+        requires_reindex=True,
+    ),
 ]

components/renku_data_services/solr/solr_schema.py

@@ -80,6 +80,11 @@ class Filters:
     english_minimal_stem = Filter(name="englishMinimalStem")
     classic = Filter(name="classic")
     ngram = Filter(name="nGram")
+    flattenGraph = Filter(name="flattenGraph")
+    word = Filter(
+        name="wordDelimiterGraph",
+        settings={"splitOnCaseChange": "1", "catenateNumbers": "1", "catenateAll": "1", "preserveOriginal": "1"},
+    )
 
     @classmethod
     def edgeNgram(cls, min_gram_size: int = 3, maxGramSize: int = 6, preserve_original: bool = True) -> Filter:

test/bases/renku_data_services/data_api/test_search.py

@@ -292,6 +292,58 @@ async def test_search_by_entity_type(
                 assert item.root.type == field.value
 
 
+# TODO: figure out how to run search tests fully parallel
+@pytest.mark.xdist_group("search")
+@pytest.mark.asyncio
+async def test_search_project_with_dash(
+    create_project_model: CreateProjectCall,
+    regular_user: UserInfo,
+    search_query: SearchQueryCall,
+    search_reprovision: SearchReprovisionCall,
+    sanic_client_with_solr: SanicASGITestClient,
+    app_manager_instance: TestDependencyManager,
+) -> None:
+    p1 = await create_project_model(sanic_client_with_solr, "project-with-dash")
+    await search_reprovision(app_manager_instance)
+
+    result = await search_query(sanic_client_with_solr, "project-with-dash", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "project-with-hash", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+
+
+# TODO: figure out how to run search tests fully parallel
+@pytest.mark.xdist_group("search")
+@pytest.mark.asyncio
+async def test_search_domain_and_email(
+    create_project_model: CreateProjectCall,
+    regular_user: UserInfo,
+    search_query: SearchQueryCall,
+    search_reprovision: SearchReprovisionCall,
+    sanic_client_with_solr: SanicASGITestClient,
+    app_manager_instance: TestDependencyManager,
+) -> None:
+    p1 = await create_project_model(
+        sanic_client_with_solr,
+        "bob's project",
+        description="Bob's e-mail address is bob@microsoft.com - keep it safe. Check the homepage at https://bob.me/about!",
+    )
+    await search_reprovision(app_manager_instance)
+
+    result = await search_query(sanic_client_with_solr, "bob@microsoft.com", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "https://bob.me/about", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "bob.me", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "e-mail", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "email", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+    result = await search_query(sanic_client_with_solr, "bob's project", regular_user)
+    assert_search_result(result, [p1], check_order=False)
+
+
 def __entity_id(e: SearchEntity) -> str:
     match e.root:
         case SearchProject() as p: