Strengthen search relevance ranking (stemming, phrase, synonyms, CJK)

Relevance was exact lowercase-token overlap, so "transformer" missed
"transformers", multi-word queries got no adjacency credit, acronyms
never matched their expansions, and CJK queries got no signal at all.

- conservative English stemming, min-stem >= 4 guards over-stripping
- adjacency bonus so a query's phrase outranks scattered terms
- small acronym <-> expansion synonym map (llm, rag, gnn, ...), expanding
  documents not the query so the relevance denominator stays honest
- CJK character bigrams so Chinese / Japanese / Korean queries rank too

Adds unit tests plus a golden-query regression set that pins relevance
dominance over a 180k-citation off-topic paper.

Author: JE-Chen

tests/test_dedup_ranking.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import pytest
+
 from thesisagents.core.dedup import dedupe
 from thesisagents.core.models import Paper
 from thesisagents.core.ranking import rank
@@ -193,3 +195,126 @@ def test_rank_short_query_terms_are_dropped():
     miss = _paper(source_id="miss", title="A study of birds", year=2024)
     ordered = rank([miss, hit], keywords="a llm of", current_year=2025)
     assert ordered[0].source_id == "hit"
+
+
+def test_rank_stemming_matches_plural():
+    """A singular query term matches a plural/inflected title — same concept.
+
+    Without stemming, "transformer" would miss "Transformers" entirely and the
+    on-topic paper would sort below an unrelated one on recency/citation alone.
+    """
+    hit = _paper(source_id="hit", title="Transformers in Vision", year=2024)
+    miss = _paper(source_id="miss", title="Graph Theory Basics", year=2024)
+    ordered = rank([miss, hit], keywords="transformer", current_year=2025)
+    assert ordered[0].source_id == "hit"
+
+
+def test_rank_phrase_adjacency_bonus():
+    """Two titles share all query words, but the one where they appear ADJACENT
+    (a real phrase) ranks above the one where they are scattered."""
+    phrase = _paper(
+        source_id="phrase", title="Retrieval-Augmented Generation", year=2024,
+    )
+    scattered = _paper(
+        source_id="scattered", title="Generation Augmented by Retrieval", year=2024,
+    )
+    ordered = rank(
+        [scattered, phrase],
+        keywords="retrieval augmented generation", current_year=2025,
+    )
+    assert ordered[0].source_id == "phrase"
+
+
+def test_rank_synonym_acronym_expands():
+    """A query for an acronym matches a title that only writes the long form."""
+    hit = _paper(
+        source_id="hit", title="A Survey of Large Language Models", year=2024,
+    )
+    miss = _paper(source_id="miss", title="Image Segmentation Methods", year=2024)
+    ordered = rank([miss, hit], keywords="llm", current_year=2025)
+    assert ordered[0].source_id == "hit"
+
+
+def test_rank_synonym_does_not_overmatch_shared_word():
+    """A lone shared word ("language") must NOT inject an unrelated acronym:
+    a query for "nlp" must not match a pure "language model" title."""
+    lang_only = _paper(
+        source_id="lang", title="Sign Language Recognition", year=2024,
+    )
+    nlp_paper = _paper(
+        source_id="nlp", title="Natural Language Processing Advances", year=2024,
+    )
+    ordered = rank([lang_only, nlp_paper], keywords="nlp", current_year=2025)
+    assert ordered[0].source_id == "nlp"
+
+
+def test_rank_cjk_query_has_relevance():
+    """A Chinese-keyword search gets a real relevance signal (CJK bigrams), so an
+    on-topic Chinese title beats an off-topic one rather than tying on recency."""
+    hit = _paper(source_id="hit", title="視覺注意力機制研究", year=2024)
+    miss = _paper(source_id="miss", title="區塊鏈技術概論", year=2024)
+    ordered = rank([miss, hit], keywords="注意力機制", current_year=2025)
+    assert ordered[0].source_id == "hit"
+
+
+def test_rank_stemming_does_not_overstrip():
+    """The min-stem-length guard keeps short words intact: "ring" must stay
+    "ring" (not collapse to "r"), so it still matches a "Ring ..." title and
+    "Rendering" correctly stems to "render" without colliding."""
+    hit = _paper(source_id="hit", title="Ring Signatures", year=2024)
+    miss = _paper(source_id="miss", title="Rendering Pipelines", year=2024)
+    ordered = rank([miss, hit], keywords="ring", current_year=2025)
+    assert ordered[0].source_id == "hit"
+
+
+# --- Golden-query regression set -------------------------------------------
+# A realistic candidate pool ranked end-to-end (relevance + recency + citation
+# together), so a future change to the weights / stemmer / synonyms that quietly
+# degrades real-world ranking is caught — not just the isolated-axis unit tests
+# above. The invariant each case pins: the on-topic paper takes the #1 slot even
+# though the pool contains a 180k-citation OFF-topic paper (resnet) — i.e.
+# relevance still dominates raw citation count. (Below #1, the off-topic paper
+# may legitimately fill a slot on a query where only one paper is on-topic; that
+# is correct recency+citation behaviour, so the test pins #1, not the whole top
+# 3.) The gaps are wide enough to survive small weight retunes; if one breaks,
+# the ranking changed materially and the expectation should be re-justified.
+def _golden_pool() -> list[Paper]:
+    return [
+        _paper(source_id="attn", title="Attention Is All You Need",
+               year=2017, citation_count=90000),
+        _paper(source_id="xformer_survey", title="A Survey of Transformer Architectures",
+               year=2024, citation_count=120),
+        _paper(source_id="resnet", title="Deep Residual Learning for Image Recognition",
+               year=2016, citation_count=180000),
+        _paper(source_id="rag",
+               title="Retrieval-Augmented Generation for Knowledge-Intensive NLP",
+               year=2020, citation_count=5000),
+        _paper(source_id="gpt3", title="Large Language Models are Few-Shot Learners",
+               year=2020, citation_count=30000),
+        _paper(source_id="gnn", title="Graph Neural Networks: A Comprehensive Review",
+               year=2021, citation_count=800),
+    ]
+
+
+@pytest.mark.parametrize(
+    ("query", "expected_top"),
+    [
+        # Plural/inflected title match (stemming): "transformer" -> "Transformer".
+        ("transformer", "xformer_survey"),
+        # Exact multi-word phrase wins on adjacency + full overlap.
+        ("retrieval augmented generation", "rag"),
+        # Acronym synonym: "llm" surfaces the long-form "Large Language Models".
+        ("llm", "gpt3"),
+        # Plural in title ("Networks") + phrase; on-topic beats the 180k-cite resnet.
+        ("graph neural network", "gnn"),
+    ],
+)
+def test_rank_golden_queries(query, expected_top):
+    ordered = rank(_golden_pool(), keywords=query, current_year=2026)
+    top_ids = [p.source_id for p in ordered]
+    assert top_ids[0] == expected_top, f"{query!r} -> {top_ids[:3]}"
+    # Relevance dominance: the on-topic paper outranks the 180k-citation
+    # off-topic paper, even with a ~400x citation disadvantage in some cases.
+    assert top_ids.index(expected_top) < top_ids.index("resnet"), (
+        f"off-topic resnet outranked {expected_top!r} for {query!r}: {top_ids}"
+    )

thesisagents/core/ranking.py

@@ -13,7 +13,23 @@
 * **relevance** (dominant) — overlap between the query keywords and the paper's
   title (weighted heavily) + abstract (weighted lightly). Research starts from
   "is this on my topic?", so an on-topic paper should beat an off-topic one even
-  when the off-topic one is older-and-more-cited.
+  when the off-topic one is older-and-more-cited. The relevance axis goes beyond
+  exact word matching in four ways so it does not silently miss on-topic papers:
+
+  1. **Light stemming** — a query for ``transformer`` matches a ``Transformers``
+     title (a plural/inflection difference is the same concept). Stemming is
+     deliberately conservative: only a small whitelist of suffixes is stripped,
+     and only when the remaining stem stays ``>= _MIN_STEM_LEN`` so short words
+     (``bias``, ``ring``, ``gas``) are never mangled into a false match.
+  2. **Phrase adjacency bonus** — for a multi-word query, a title where the
+     query words appear *adjacent* (``Retrieval-Augmented Generation``) scores
+     above one where they are merely *scattered* across the title.
+  3. **Acronym synonyms** — a query for ``llm`` matches a title that only ever
+     writes ``large language model`` (and vice-versa), via a small curated map.
+  4. **CJK support** — Chinese/Japanese/Korean runs are tokenised into character
+     bigrams, so a Chinese-keyword search gets a real relevance signal instead
+     of falling back to recency+citation only (the prior ``[a-z0-9]+`` tokenizer
+     dropped every CJK character).
 * **recency** — exponential decay over paper age (~5-year scale).
 * **citation** — ``log10`` of the citation count (diminishing returns), then
   damped by ``_CITATION_WEIGHT`` so a huge citation count is a strong tie-break
@@ -40,14 +56,121 @@
 # (citation term ≈ _CITATION_WEIGHT * log10(1e5) = 0.4 * 5 = 2.0 < 3.0).
 _TITLE_WEIGHT = 3.0
 _ABSTRACT_WEIGHT = 0.6
+# Bonus when the query's adjacent word pairs (bigrams) also appear adjacent in
+# the title. Smaller than _TITLE_WEIGHT so it only *re-orders* papers that
+# already share the query words — a phrase match is a tie-break in favour of the
+# more on-topic title, not a signal that can outweigh actual word overlap.
+_PHRASE_WEIGHT = 1.0
 # Damping on the log10 citation term. Keeps "most-cited" a meaningful tie-break
 # among similarly-relevant papers without letting it swamp the topic signal.
 _CITATION_WEIGHT = 0.4
 # Query/title tokens shorter than this are dropped as stop-word-ish noise
 # ("a", "of", "is", "the"). 3 keeps useful short terms like "llm", "rag", "gan".
 _MIN_TERM_LEN = 3
+# A suffix is only stripped when the remaining stem stays at least this long.
+# Guards against over-stripping short words into spurious collisions, e.g.
+# "ring" -> "r", "bias" -> "bia", "gas" -> "ga". 4 keeps stemming useful for
+# real content words ("transformers" -> "transformer", "learning" -> "learn")
+# while leaving every short word untouched.
+_MIN_STEM_LEN = 4
+
+# One regex, two alternatives: an ASCII alphanumeric run, OR a run of CJK
+# characters (CJK unified incl. Ext-A, hiragana, katakana, hangul syllables,
+# CJK compatibility ideographs). Matching both in one pass keeps token order so
+# adjacency (the phrase bonus) is computed across the original sequence.
+_TOKEN_RE = re.compile(
+    r"[a-z0-9]+"
+    r"|[぀-ヿ㐀-鿿가-힯豈-﫿]+"
+)
+
+# Conservative English suffix rules, longest/most-specific first so "ies" wins
+# over "s" ("studies" -> "study", not "studie"). Each maps a suffix to its
+# replacement. Only applied when the resulting stem stays >= _MIN_STEM_LEN.
+_SUFFIX_RULES: tuple[tuple[str, str], ...] = (
+    ("ies", "y"),
+    ("es", ""),
+    ("ed", ""),
+    ("ing", ""),
+    ("s", ""),
+)
+
+# Acronym <-> expansion synonyms. Each entry lets a search for the short form
+# surface papers that only write the long form, and vice-versa.
+# Why: without this the relevance axis misses a whole class of on-topic papers
+# (a "llm" search never matching a "Large Language Models: A Survey" title).
+# Only acronyms of length >= _MIN_TERM_LEN are listed — shorter ones ("rl",
+# "ml") would be dropped by the stop-word floor before they could be matched.
+_SYNONYM_GROUPS: tuple[tuple[str, str], ...] = (
+    ("llm", "large language model"),
+    ("rag", "retrieval augmented generation"),
+    ("gnn", "graph neural network"),
+    ("cnn", "convolutional neural network"),
+    ("rnn", "recurrent neural network"),
+    ("nlp", "natural language processing"),
+    ("vlm", "vision language model"),
+    ("gan", "generative adversarial network"),
+)
+
+
+def _stem(token: str) -> str:
+    """Conservatively normalise one token's English inflection.
+
+    CJK bigrams, digits, and mixed alphanumerics pass through unchanged — only
+    pure ASCII alphabetic tokens are stemmed, and only when the stem stays
+    ``>= _MIN_STEM_LEN``. A trailing "ss" (``process``, ``address``) is left
+    alone so the plural "s" rule does not bite into a doubled consonant.
+
+    Example: ``_stem("transformers") == "transformer"``;
+    ``_stem("ring") == "ring"`` (stripping "ing" -> "r" fails the length guard).
+    """
+    if not token.isascii() or not token.isalpha():
+        return token
+    for suffix, repl in _SUFFIX_RULES:
+        if suffix == "s" and token.endswith("ss"):
+            continue
+        if token.endswith(suffix):
+            stem = token[: len(token) - len(suffix)] + repl
+            return stem if len(stem) >= _MIN_STEM_LEN else token
+    return token
 
-_WORD_RE = re.compile(r"[a-z0-9]+")
+
+def _ordered_tokens(text: str) -> list[str]:
+    """Position-ordered, stemmed, stop-word-filtered tokens of ``text``.
+
+    ASCII runs become stemmed words kept only when ``len >= _MIN_TERM_LEN``; CJK
+    runs become character bigrams (each length 2, always kept). Order is
+    preserved across scripts so the bigram (phrase) pass sees real adjacency.
+    """
+    out: list[str] = []
+    for match in _TOKEN_RE.finditer(text.lower()):
+        chunk = match.group()
+        if chunk[0].isascii():
+            stem = _stem(chunk)
+            if len(stem) >= _MIN_TERM_LEN:
+                out.append(stem)
+        elif len(chunk) == 1:
+            out.append(chunk)
+        else:
+            out.extend(chunk[i : i + 2] for i in range(len(chunk) - 1))
+    return out
+
+
+# Acronym -> stemmed long-form tokens. One direction only: an acronym is
+# specific, so seeing "llm" in a document safely implies "large language model".
+# The REVERSE (long form -> acronym) deliberately is NOT a per-token map — a lone
+# shared word like "language" must not inject "nlp"/"vlm"; it requires the whole
+# long form and is handled by _SYNONYM_LONG_TO_SHORT below.
+_SYNONYM_EXPAND: dict[str, tuple[str, ...]] = {
+    short: tuple(_ordered_tokens(long_form))
+    for short, long_form in _SYNONYM_GROUPS
+}
+# Whole stemmed long forms -> acronym: the acronym is added to a document's term
+# set only when every token of the long form is present (so "Large Language
+# Models" gains "llm", but "language models" alone does not).
+_SYNONYM_LONG_TO_SHORT: tuple[tuple[frozenset[str], str], ...] = tuple(
+    (frozenset(_ordered_tokens(long_form)), short)
+    for short, long_form in _SYNONYM_GROUPS
+)
 
 
 def rank(
@@ -62,38 +185,72 @@ def rank(
     axis (single-paper / query-less callers).
     """
     year_base = current_year or _CURRENT_YEAR_FALLBACK
-    terms = _terms(keywords) if keywords else frozenset()
+    terms = frozenset(_ordered_tokens(keywords)) if keywords else frozenset()
+    bigrams = _bigrams(keywords) if keywords else frozenset()
     return sorted(
         papers,
-        key=lambda paper: _score(paper, year_base, terms),
+        key=lambda paper: _score(paper, year_base, terms, bigrams),
         reverse=True,
     )
 
 
-def _score(paper: Paper, current_year: int, terms: frozenset[str]) -> float:
+def _score(
+    paper: Paper,
+    current_year: int,
+    terms: frozenset[str],
+    query_bigrams: frozenset[tuple[str, str]],
+) -> float:
     return (
-        _relevance_score(paper, terms)
+        _relevance_score(paper, terms, query_bigrams)
         + _recency_score(paper.year, current_year)
         + _citation_score(paper.citation_count)
     )
 
 
-def _terms(text: str) -> frozenset[str]:
-    """Lowercase alphanumeric tokens of length >= ``_MIN_TERM_LEN``."""
-    return frozenset(w for w in _WORD_RE.findall(text.lower()) if len(w) >= _MIN_TERM_LEN)
+def _bigrams(text: str) -> frozenset[tuple[str, str]]:
+    """Adjacent token pairs of ``text`` (empty when fewer than two tokens)."""
+    tokens = _ordered_tokens(text)
+    return frozenset(zip(tokens, tokens[1:], strict=False))
+
+
+def _term_set(text: str) -> frozenset[str]:
+    """Synonym-expanded term set of a document field (title / abstract).
+
+    Documents — not the query — are expanded, so the relevance denominator stays
+    the user's actual query size (expanding the query would dilute the fraction).
+    """
+    base = set(_ordered_tokens(text))
+    expanded = set(base)
+    for token in base:
+        expanded.update(_SYNONYM_EXPAND.get(token, ()))
+    for long_tokens, short in _SYNONYM_LONG_TO_SHORT:
+        if long_tokens <= base:
+            expanded.add(short)
+    return frozenset(expanded)
 
 
-def _relevance_score(paper: Paper, terms: frozenset[str]) -> float:
-    """Fraction of query terms appearing in title / abstract, title-weighted.
+def _relevance_score(
+    paper: Paper,
+    terms: frozenset[str],
+    query_bigrams: frozenset[tuple[str, str]],
+) -> float:
+    """Title/abstract overlap with the query, title-weighted, plus a phrase bonus.
 
-    Range ``[0, _TITLE_WEIGHT + _ABSTRACT_WEIGHT]``. ``0.0`` when no query terms
-    were supplied, so the score reduces to recency + citation.
+    Range ``[0, _TITLE_WEIGHT + _ABSTRACT_WEIGHT + _PHRASE_WEIGHT]``. ``0.0``
+    when no query terms were supplied, so the score reduces to recency +
+    citation.
     """
     if not terms:
         return 0.0
-    title_hit = len(terms & _terms(paper.title)) / len(terms)
-    abstract_hit = len(terms & _terms(paper.abstract)) / len(terms)
-    return title_hit * _TITLE_WEIGHT + abstract_hit * _ABSTRACT_WEIGHT
+    title_terms = _term_set(paper.title)
+    abstract_terms = _term_set(paper.abstract)
+    title_hit = len(terms & title_terms) / len(terms)
+    abstract_hit = len(terms & abstract_terms) / len(terms)
+    score = title_hit * _TITLE_WEIGHT + abstract_hit * _ABSTRACT_WEIGHT
+    if query_bigrams:
+        matched = len(query_bigrams & _bigrams(paper.title))
+        score += _PHRASE_WEIGHT * (matched / len(query_bigrams))
+    return score
 
 
 def _recency_score(year: int | None, current_year: int) -> float: