Merge remote-tracking branch 'origin/develop' into feat/atr-detection-rail

Signed-off-by: eeee2345 <217509886+eeee2345@users.noreply.github.com>

Author: eeee2345

.github/workflows/_test.yml

@@ -96,11 +96,11 @@ jobs:
 
       - name: Run pytest with coverage
         if: inputs.with-coverage
-        run: poetry run pytest --cov=nemoguardrails tests/ --cov-report=xml:coverage.xml -v
+        run: make test-coverage
 
       - name: Run pytest without coverage
         if: inputs.with-coverage == false
-        run: poetry run pytest -v
+        run: make test
 
       - name: Upload coverage to Codecov
         if: inputs.with-coverage

.github/workflows/lint.yml

@@ -55,4 +55,4 @@ jobs:
         run: poetry install --with dev
 
       - name: Run pre-commit hooks
-        run: poetry run make pre_commit
+        run: poetry run make pre-commit

Makefile

@@ -1,25 +1,48 @@
-.PHONY: all test tests test_watch test_coverage test_profile docs docs-strict docs-serve docs-update-cards docs-check-cards docs-watch-cards pre_commit help
+.PHONY: help
+.PHONY: test test-parallel test-serial test-benchmark test-watch test-coverage test-profile warm-fastembed-cache
+.PHONY: docs docs-strict docs-serve docs-update-cards docs-check-cards docs-watch-cards docs-check-redirects
+.PHONY: pre-commit
 
-# Default target executed when no specific target is provided to make.
-all: help
+.DEFAULT_GOAL := help
 
-# Define a variable for the test file path.
-TEST_FILE ?= tests/
+TEST ?=
+ARGS ?=
+WORKERS ?= auto
+# pytest-xdist --dist strategy for $(PYTEST) -n $(WORKERS) --dist $(DIST) $(ARGS) $(TEST).
+# worksteal dynamically rebalances queued tests; override DIST when debugging or grouping matters.
+DIST ?= worksteal
+
+PYTEST ?= poetry run pytest
+# These targets assume a Unix-like shell for env -u; use bash, Git Bash, or WSL on Windows.
+UNIT_TEST_ENV ?= env -u OPENAI_API_KEY -u NVIDIA_API_KEY \
+	-u LIVE_TEST -u LIVE_TEST_MODE -u TEST_LIVE_MODE
+
+FASTEMBED_CACHE ?= .cache/fastembed
+FASTEMBED_MODEL ?= sentence-transformers/all-MiniLM-L6-v2
+FASTEMBED_ENV ?= env FASTEMBED_CACHE_PATH=$(FASTEMBED_CACHE)
 
 test:
-	poetry run pytest $(TEST_FILE)
+	$(UNIT_TEST_ENV) $(PYTEST) -n $(WORKERS) --dist $(DIST) $(ARGS) $(TEST)
+
+test-parallel: test
+
+test-serial:
+	$(PYTEST) $(ARGS) $(TEST)
 
-tests:
-	poetry run pytest $(TEST_FILE)
+test-benchmark:
+	$(PYTEST) $(ARGS) benchmark/tests
 
-test_watch:
-	poetry run ptw --snapshot-update --now . -- -vv $(TEST_FILE)
+test-watch:
+	poetry run ptw --snapshot-update --now . -- -vv $(ARGS) $(TEST)
 
-test_coverage:
-	poetry run pytest --cov=$(TEST_FILE) --cov-report=term-missing
+test-coverage:
+	$(UNIT_TEST_ENV) $(PYTEST) -n $(WORKERS) --dist $(DIST) --cov=nemoguardrails --cov-report=xml:coverage.xml $(ARGS) $(TEST)
 
-test_profile:
-	poetry run pytest -vv tests/ --profile-svg
+test-profile:
+	$(PYTEST) -vv --profile-svg $(ARGS) $(TEST)
+
+warm-fastembed-cache:
+	$(FASTEMBED_ENV) poetry run python -c 'from fastembed import TextEmbedding; model = TextEmbedding("$(FASTEMBED_MODEL)"); next(model.embed(["warmup"]))'
 
 docs:
 	poetry run sphinx-build -b html docs _build/docs
@@ -42,25 +65,38 @@ docs-watch-cards:
 docs-check-redirects:
 	cd docs && poetry run python scripts/validate_redirects.py
 
-pre_commit:
-	pre-commit install
-	pre-commit run --all-files
-
-
-# HELP
+pre-commit:
+	poetry run pre-commit install
+	poetry run pre-commit run --all-files
 
 help:
-	@echo '----'
-	@echo 'test                         - run unit tests'
-	@echo 'tests                        - run unit tests'
-	@echo 'test TEST_FILE=<test_file>   - run all tests in given file'
-	@echo 'test_watch                   - run unit tests in watch mode'
-	@echo 'test_coverage                - run unit tests with coverage'
-	@echo 'docs                         - build docs, if you installed the docs dependencies'
-	@echo 'docs-strict                  - build docs with warnings as errors (used in CI)'
-	@echo 'docs-serve                   - serve docs locally with auto-rebuild on changes'
-	@echo 'docs-update-cards            - update grid cards in index files from linked pages'
-	@echo 'docs-check-cards             - check if grid cards are up to date (dry run)'
-	@echo 'docs-watch-cards             - watch for file changes and auto-update cards'
-	@echo 'docs-check-redirects         - validate that all redirect targets exist'
-	@echo 'pre_commit                   - run pre-commit hooks'
+	@printf '%s\n' \
+		'' \
+		'Usage:' \
+		'  make test [TEST=path] [WORKERS=auto] [ARGS="-q --tb=short"]' \
+		'  make test-serial [TEST=path] [ARGS="-q"]' \
+		'  make test-benchmark [ARGS="-q"]' \
+		'  make test-parallel [TEST=path] [WORKERS=auto] [ARGS="-q --tb=short"]' \
+		'  make test-watch [TEST=path]' \
+		'' \
+		'Tests:' \
+		'  test                  Run pytest.ini testpaths with pytest-xdist' \
+		'  test-parallel         Alias for test' \
+		'  test-serial           Run pytest without xdist or env filtering' \
+		'  test-benchmark        Run benchmark tooling tests' \
+		'  test-watch            Run pytest in watch mode' \
+		'  test-coverage         Run pytest with coverage' \
+		'  test-profile          Run pytest with profiling' \
+		'  warm-fastembed-cache  Prime the repo-local FastEmbed cache' \
+		'' \
+		'Docs:' \
+		'  docs                  Build docs' \
+		'  docs-strict           Build docs with warnings as errors' \
+		'  docs-serve            Serve docs locally' \
+		'  docs-update-cards     Update generated docs cards' \
+		'  docs-check-cards      Check generated docs cards' \
+		'  docs-watch-cards      Watch and update generated docs cards' \
+		'  docs-check-redirects  Validate docs redirects' \
+		'' \
+		'Maintenance:' \
+		'  pre-commit            Install and run pre-commit hooks'

poetry.lock

@@ -164,6 +164,7 @@ pytest = ">=7.2.2,<9.0.0"
 pytest-asyncio = ">=0.21.0, <1.0.0"
 pytest-cov = ">=4.1.0"
 pytest-httpx = ">=0.22.0"
+pytest-xdist = "^3.8.0"
 streamlit = ">=1.37.0"
 tox = "^4.23.2"
 pytest-profiling = "^1.7.0"

pyproject.toml

@@ -10,6 +10,12 @@ log_cli = False
 
 asyncio_default_fixture_loop_scope = function
 
+markers =
+    serial: documentation-only marker for tests that may need serial scheduling
+    slow: high-runtime tests that may need separate scheduling
+    live: tests that require real provider credentials or external services
+    real_embeddings: tests that intentionally exercise configured real embedding providers
+
 testpaths =
     tests
     docs/colang-2/examples

pytest.ini

@@ -13,16 +13,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from unittest.mock import patch
 
 import pytest
 
+from tests.testing.embeddings import (
+    DeterministicEmbeddingSearchProvider,
+)
+
 try:
     import langchain_core  # noqa: F401
 except ImportError:
     collect_ignore_glob = ["integrations/langchain/*.py", "integrations/langchain/**/*.py"]
 
 REASONING_TRACE_MOCK_PATH = "nemoguardrails.actions.llm.generation.get_and_clear_reasoning_trace_contextvar"
+_FASTEMBED_ENGINES = {"FastEmbed"}
+_FASTEMBED_MODELS = {"all-MiniLM-L6-v2", "sentence-transformers/all-MiniLM-L6-v2"}
+_REAL_EMBEDDING_ENV_VARS = (
+    "LIVE_TEST",
+    "LIVE_TEST_MODE",
+    "TEST_LIVE_MODE",
+)
 
 
 @pytest.fixture(autouse=True)
@@ -35,6 +47,53 @@ def reset_reasoning_trace_var():
     reasoning_trace_var.set(None)
 
 
+@pytest.fixture(autouse=True)
+def reset_tool_calls_var():
+    from nemoguardrails.context import tool_calls_var
+
+    tool_calls_var.set(None)
+    yield
+    tool_calls_var.set(None)
+
+
+@pytest.fixture(autouse=True)
+def reset_explain_info_var():
+    from nemoguardrails.context import explain_info_var
+
+    explain_info_var.set(None)
+    yield
+    explain_info_var.set(None)
+
+
+@pytest.fixture(autouse=True)
+def use_deterministic_embeddings_for_default_fastembed(monkeypatch, request):
+    if request.node.get_closest_marker("real_embeddings") or any(
+        os.environ.get(name) for name in _REAL_EMBEDDING_ENV_VARS
+    ):
+        return
+
+    from nemoguardrails.rails.llm.llmrails import LLMRails
+
+    original_get_embedding_search_provider = LLMRails._get_embeddings_search_provider_instance
+
+    def patched_get_embedding_search_provider(self, esp_config=None):
+        if esp_config is None or _uses_default_fastembed(
+            esp_config,
+            self.default_embedding_engine,
+            self.default_embedding_model,
+        ):
+            parameters = getattr(esp_config, "parameters", {}) if esp_config is not None else {}
+            return DeterministicEmbeddingSearchProvider(**parameters)
+
+        return original_get_embedding_search_provider(self, esp_config)
+
+    monkeypatch.setattr(
+        LLMRails,
+        "_get_embeddings_search_provider_instance",
+        patched_get_embedding_search_provider,
+    )
+
+
 @pytest.fixture
 def langchain_framework():
     from nemoguardrails.llm.frameworks import _reset_frameworks, set_default_framework
@@ -47,3 +106,13 @@ def langchain_framework():
 
 def pytest_configure(config):
     patch("prompt_toolkit.PromptSession", autospec=True).start()
+
+
+def _uses_default_fastembed(provider_config, default_engine, default_model):
+    if provider_config.name != "default":
+        return False
+
+    parameters = provider_config.parameters
+    engine = parameters.get("embedding_engine", default_engine)
+    model = parameters.get("embedding_model", default_model)
+    return engine in _FASTEMBED_ENGINES and model in _FASTEMBED_MODELS