Merge branch 'main' into feature/vector-range-search

Author: AlexFilipImproving

.gitignore

@@ -21,3 +21,4 @@ venv/
 env/
 .venv/
 .env/
+.build_log

docs/commands/ft.create.md

@@ -12,6 +12,7 @@ FT.CREATE <index-name>
     [ON HASH | ON JSON]
     [PREFIX <count> <prefix> [<prefix>...]]
     [SCORE default_value]
+    [SCORE_FIELD <field_name>]
     [LANGUAGE <language>]
     [SKIPINITIALSCAN]
     [MINSTEMSIZE <min_stem_size>]
@@ -47,7 +48,9 @@ FT.CREATE <index-name>
 
 - `SKIPINITIALSCAN` (optional): If specified, this option skips the normal backfill operation for an index. If this option is specified, pre-existing keys which match the `PREFIX` clause will not be loaded into the index during a backfill operation. This clause has no effect on processing of key mutations _after_ an index is created, i.e., keys which are mutated after an index is created and satisfy the data type and `PREFIX` clause will be inserted into that index.
 
-- `SCORE` (optional): The current implementation only allows the value to be 1.0. This parameter is accepted to make valkey-search more interoperable with RediSearch. (default: 1.0)
+- `SCORE` (optional): Sets the default document score used for text search ranking. The value must be between 0.0 and 1.0. When `SCORE_FIELD` is configured, this value is used as the fallback if a document's score field is missing or cannot be parsed. (default: 1.0)
+
+- `SCORE_FIELD <field_name>` (optional): Specifies the name of a hash field whose numeric value is used as the per-document score. When configured, the value of this field is read during ingestion and stored as the document's relevance score for text search ranking. If the field is missing or cannot be parsed as a valid number, the index-level `SCORE` default is used. The raw value is stored without clamping; the scoring algorithm determines how to handle values at query time.
 
 ## Field types
 

integration/compatibility/README

@@ -1,5 +1,27 @@
-To generate the compatibility suite, cd to this directory and enter:
+To regenerate both compatibility pickle answer files, from the repo root run:
 
-pytest generate.py
+    ./integration/compatibility/regenerate.sh
 
-The current handling of the docker instance is flaky, sometimes it needs to be manually killed.
+Or, to regenerate just one, cd to this directory and run pytest directly:
+
+    pytest generate.py        # produces aggregate-answers.pickle.gz
+    pytest generate_text.py   # produces text-search-answers.pickle.gz
+
+Both forms require Docker. The generators spin up redis/redis-stack-server on
+port 6380 to capture reference answers; the docker handling is sometimes flaky
+and may need to be killed manually between runs.
+
+Each pickle stores a SHA256 of every .py file in this directory. The
+compatibility integration test (integration/compatibility_test.py) verifies
+that hash on load and fails if it does not match the current sources -- this
+forces a regeneration whenever generate.py, generate_text.py, data_sets.py,
+text_query_builder.py, or any other source here is edited.
+
+To add a new generator: create generate_xxx.py (subclass BaseCompatibilityTest
+with its own ANSWER_FILE_NAME) and add an entry to the GENERATORS list in
+__init__.py. regenerate.sh and compatibility_test.py both read from that list.
+
+To bypass the hash check (e.g. when generating a small pickle locally for
+quick iteration), set:
+
+    SKIP_COMPATIBILITY_HASH_CHECK=1

integration/compatibility/__init__.py

@@ -1,3 +1,32 @@
-#
-# Make this a module
-#
+import hashlib
+import os
+
+_COMPAT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+# Registry of compatibility generators. To add a new generator, create the
+# generate file (subclassing BaseCompatibilityTest with its own
+# ANSWER_FILE_NAME) and add an entry here. regenerate.sh and
+# compatibility_test.py both read from this list.
+GENERATORS = [
+    {"generator": "generate.py",      "answers": "aggregate-answers.pickle.gz",   "cluster": True},
+    {"generator": "generate_text.py", "answers": "text-search-answers.pickle.gz", "cluster": False},
+]
+
+
+def compute_sources_hash():
+    """SHA256 of every .py file in this directory.
+
+    Stored inside the generated pickle answer files so compatibility_test.py
+    can detect when a pickle is stale relative to the generators and helpers.
+    """
+    h = hashlib.sha256()
+    for fname in sorted(os.listdir(_COMPAT_DIR)):
+        if not fname.endswith(".py"):
+            continue
+        h.update(fname.encode("utf-8"))
+        h.update(b"\0")
+        with open(os.path.join(_COMPAT_DIR, fname), "rb") as f:
+            h.update(f.read())
+        h.update(b"\0")
+    return h.hexdigest()

integration/compatibility/generate.py

@@ -4,6 +4,7 @@
 import gzip
 from . import data_sets
 from .data_sets import *
+from . import compute_sources_hash
 from valkey.exceptions import ConnectionError
 '''
 Capture answer from Redisearch
@@ -66,8 +67,12 @@ def teardown_class(cls):
         print("Stopping Generate-search server")
         os.system("docker stop Generate-search")
         print("Dumping ", len(cls.answers), " answers")
+        payload = {
+            "sources_hash": compute_sources_hash(),
+            "answers": cls.answers,
+        }
         with gzip.open(cls.ANSWER_FILE_NAME, "wb") as answer_file:
-            pickle.dump(cls.answers, answer_file)
+            pickle.dump(payload, answer_file)
 
     def setup_method(self):
         self.client.execute_command("FLUSHALL SYNC")
@@ -186,22 +191,24 @@ def checkall(self, dialect, *orig_cmd, **kwargs):
         self.checkvec(dialect, *orig_cmd, **kwargs)
         self.check(dialect, *orig_cmd)
 
+    @pytest.mark.skip(reason="Needs fix for ingesting invalid data")
     def test_bad_numeric_data(self, key_type, dialect):
         self.setup_data("bad numbers", key_type)
         self.check(dialect, "ft.search", f"{key_type}_idx1", "@n1:[-inf inf]")
         self.check(dialect, "ft.search", f"{key_type}_idx1", "-@n1:[-inf inf]")
         self.check(dialect, "ft.search", f"{key_type}_idx1", "@n2:[-inf inf]")
         self.check(dialect, "ft.search", f"{key_type}_idx1", "-@n2:[-inf inf]")
 
+    @pytest.mark.skip(reason="Needs research")
     def test_search_reverse(self, key_type, dialect):
         self.setup_data("reverse vector numbers", key_type)
         self.checkall(dialect, f"ft.search {key_type}_idx1 *")
         self.checkall(dialect, f"ft.search {key_type}_idx1 * limit 0 5")
 
+    @pytest.mark.skip(reason="Needs research")
     def test_search(self, key_type, dialect):
         self.setup_data("sortable numbers", key_type)
         self.checkall(dialect, f"ft.search {key_type}_idx1 *")
-        self.checkall(dialect, f"ft.search {key_type}_idx1 * limit 0 5")
 
     @pytest.mark.parametrize("algo", ["flat", "hnsw"])
     @pytest.mark.parametrize("metric", ["l2", "ip", "cosine"])
@@ -490,7 +497,7 @@ def test_aggregate_dyadic_ops(self, key_type, dialect):
                         "as",
                         "nn",
                 )
-
+    @pytest.mark.skip(reason="Needs research")
     def test_search_sortby(self, key_type, dialect):
         self.setup_data("sortable numbers", key_type)
 
@@ -617,4 +624,3 @@ def test_vector_range_epsilon(self, key_type, dialect):
             radius=50, query_vector=[0] * VECTOR_DIM,
             query_attrs="{$epsilon: 0.5}",
         )
-

integration/compatibility/generate_text.py

@@ -399,26 +399,30 @@ def test_text_search_group_depth2(self, key_type, dialect, schema_type):
     def test_text_search_group_depth3(self, key_type, dialect, schema_type):
         """Test grouped queries with depth 3."""
         self._run_test(gen_depth3, "pure text", key_type, dialect, schema_type)
-    
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth2_inorder(self, key_type, dialect, schema_type):
         """Test grouped queries with depth 2."""
         self._run_test(gen_depth2, "pure text", key_type, dialect, schema_type, inorder=True, check_parsing=True)
 
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth3_inorder(self, key_type, dialect, schema_type):
         """Test grouped queries with depth 3."""
         self._run_test(gen_depth3, "pure text", key_type, dialect, schema_type, inorder=True, check_parsing=True)
-    
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth2_slop(self, key_type, dialect, schema_type):
         """Test grouped queries with depth 2."""
         self._run_test(gen_depth2, "pure text", key_type, dialect, schema_type, slop=True, check_parsing=True)
 
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth3_slop(self, key_type, dialect, schema_type):
         """Test grouped queries with depth 3."""
         self._run_test(gen_depth3, "pure text", key_type, dialect, schema_type, slop=True, check_parsing=True)
 
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth2_inorder_slop(self, key_type, dialect, schema_type):
         self._run_test(gen_depth2, "pure text", key_type, dialect, schema_type, inorder=True, slop=True, check_parsing=True)
 
+    @pytest.mark.skip(reason="Not sure when these got broken")
     def test_text_search_group_depth3_inorder_slop(self, key_type, dialect, schema_type):
         self._run_test(gen_depth3, "pure text", key_type, dialect, schema_type, inorder=True, slop=True, check_parsing=True)
 

integration/compatibility/regenerate.sh

@@ -0,0 +1,65 @@
+#!/bin/bash -e
+# Regenerate the compatibility test pickle answer files
+# (aggregate-answers.pickle.gz and text-search-answers.pickle.gz).
+#
+# Requires Docker: the generators spin up redis/redis-stack-server on port 6380
+# to capture reference answers.
+#
+# Usage:
+#   ./integration/compatibility/regenerate.sh [extra pytest args...]
+#
+# After it finishes, git add and commit the updated *.pickle.gz files.
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+COMPAT_DIR=${ROOT_DIR}/integration/compatibility
+
+if ! command -v docker >/dev/null 2>&1; then
+    echo "ERROR: docker is required to regenerate pickle files." >&2
+    exit 1
+fi
+
+# Prefer the integration test venv (created by integration/run.sh) if present;
+# it already has pytest and the valkey client installed.
+PYTHON=""
+for build_dir in .build-release .build-debug \
+                 .build-release-asan .build-debug-asan \
+                 .build-release-tsan .build-debug-tsan; do
+    candidate="${ROOT_DIR}/${build_dir}/integration/env/bin/python3"
+    if [ -x "${candidate}" ]; then
+        PYTHON="${candidate}"
+        break
+    fi
+done
+PYTHON=${PYTHON:-python3}
+
+echo "Using python: ${PYTHON}"
+cd "${ROOT_DIR}"
+
+# Source the generator list from compatibility/__init__.py so adding a new
+# generator only requires editing one place.
+GENERATOR_FILES=()
+while IFS= read -r line; do
+    GENERATOR_FILES+=("${line}")
+done < <(PYTHONPATH=integration "${PYTHON}" -c \
+    "from compatibility import GENERATORS
+for g in GENERATORS: print(g['generator'])")
+
+ANSWER_FILES=()
+while IFS= read -r line; do
+    ANSWER_FILES+=("${line}")
+done < <(PYTHONPATH=integration "${PYTHON}" -c \
+    "from compatibility import GENERATORS
+for g in GENERATORS: print(g['answers'])")
+
+cd "${COMPAT_DIR}"
+for gen in "${GENERATOR_FILES[@]}"; do
+    echo "==> Running ${gen}"
+    "${PYTHON}" -m pytest "${gen}" "$@"
+done
+
+echo
+echo "Done. Updated files:"
+ls -la "${ANSWER_FILES[@]}"
+echo
+echo "Don't forget to 'git add' and commit them."

integration/compatibility/text-search-answers.pickle.gz

@@ -5,7 +5,11 @@
 from itertools import chain, combinations
 import pickle
 import compatibility
-from compatibility.data_sets import * 
+from compatibility import GENERATORS, compute_sources_hash
+from compatibility.data_sets import *
+
+ALL_ANSWER_FILES = [g["answers"] for g in GENERATORS]
+CLUSTER_ANSWER_FILES = [g["answers"] for g in GENERATORS if g["cluster"]]
 TEST_MARKER = "*" * 100
 from valkey_search_test_case import (
     ValkeySearchClusterTestCase,
@@ -504,8 +508,51 @@ def do_answer_cluster(cluster_client, expected, data_set, test_case):
 
     return data_set
 
+def _load_answers_with_hash_check(answer_file_name):
+    """Load a compatibility pickle answer file and verify its sources hash.
+
+    Set SKIP_COMPATIBILITY_HASH_CHECK=1 to bypass the hash check (useful when
+    manually generating a small pickle for local testing).
+    """
+    pickle_path = os.path.join(
+        os.getenv("ROOT_DIR"), "integration/compatibility", answer_file_name
+    )
+    with gzip.open(pickle_path, "rb") as f:
+        payload = pickle.load(f)
+
+    if isinstance(payload, dict) and "answers" in payload:
+        stored_hash = payload.get("sources_hash")
+        answers = payload["answers"]
+    else:
+        stored_hash = None
+        answers = payload
+
+    if os.getenv("SKIP_COMPATIBILITY_HASH_CHECK") == "1":
+        print(f"SKIP_COMPATIBILITY_HASH_CHECK=1; skipping hash check for {answer_file_name}")
+        return answers
+
+    current_hash = compute_sources_hash()
+    if stored_hash != current_hash:
+        pytest.fail(
+            f"\nCompatibility pickle file '{answer_file_name}' is stale.\n"
+            f"  Stored hash:  {stored_hash}\n"
+            f"  Current hash: {current_hash}\n"
+            f"\n"
+            f"Python sources in integration/compatibility/ have changed since\n"
+            f"the pickle was generated. Regenerate with:\n"
+            f"\n"
+            f"  ./integration/compatibility/regenerate.sh\n"
+            f"\n"
+            f"Then commit the updated pickle file. To bypass this check (e.g.\n"
+            f"when manually generating a small pickle for local testing), set\n"
+            f"the env variable SKIP_COMPATIBILITY_HASH_CHECK=1.\n",
+            pytrace=False,
+        )
+    return answers
+
+
 class TestAnswersCMD(ValkeySearchTestCaseBase):
-    @pytest.mark.parametrize("answers", ["aggregate-answers.pickle.gz", "text-search-answers.pickle.gz"])
+    @pytest.mark.parametrize("answers", ALL_ANSWER_FILES)
     def test_answers(self, answers):
         global client, data_set
         global correct_answers, failed_tests, passed_tests
@@ -517,8 +564,7 @@ def test_answers(self, answers):
         passed_tests = {}
 
         print("Running test_answers with answers file:", answers)
-        with gzip.open(os.getenv("ROOT_DIR") + "/integration/compatibility/" + answers, "rb") as answer_file:
-            answers = pickle.load(answer_file)
+        answers = _load_answers_with_hash_check(answers)
 
         data_set = None
         client = self.server.get_new_client()
@@ -565,7 +611,7 @@ def test_answers(self, answers):
 
 # TODO: fix cluster mode test failures
 class TestAnswersCME(ValkeySearchClusterTestCase):
-    @pytest.mark.parametrize("answers", ["aggregate-answers.pickle.gz"])
+    @pytest.mark.parametrize("answers", CLUSTER_ANSWER_FILES)
     def test_answers(self, answers):
         global correct_answers, wrong_answers, failed_tests, passed_tests
 
@@ -576,11 +622,7 @@ def test_answers(self, answers):
 
         print("Running CLUSTER test_answers with answers file:", answers)
 
-        with gzip.open(
-            os.getenv("ROOT_DIR") + "/integration/compatibility/" + answers,
-            "rb",
-        ) as answer_file:
-            answers = pickle.load(answer_file)
+        answers = _load_answers_with_hash_check(answers)
 
         data_set = None
         cluster_client = self.new_cluster_client()

integration/compatibility_test.py

@@ -83,6 +83,23 @@
           }
         ]
       },
+      {
+        "name": "SCORE_FIELD",
+        "type": "block",
+        "optional": true,
+        "description": "Document field whose numeric value is used as the document's custom score",
+        "arguments": [
+          {
+            "name": "score_field_token",
+            "type": "pure-token",
+            "token": "SCORE_FIELD"
+          },
+          {
+            "name": "field_name",
+            "type": "string"
+          }
+        ]
+      },
       {
         "name": "LANGUAGE",
         "type": "block",