From a11c8756e09a720e393f3eee60ac4ca1c6f3ec60 Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Thu, 11 Jun 2026 14:37:06 -0700 Subject: [PATCH 01/21] docs: document Fern docs publishing workflow (#2015) --- CONTRIBUTING.md | 26 ++++++++++++++++---------- Makefile | 7 ++++++- docs/LIVE_DOCS.mdx | 10 ++++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 090c14fbd5..294b128c60 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,6 +112,12 @@ Use the following targets for common Fern documentation tasks: | `make docs-fern-live` | Serve the Fern docs locally. | | `make docs-fern-preview-watch` | Watch local changes and publish a Fern preview for the current branch. | | `make docs-fern-generate-sdk` | Regenerate the Python SDK reference pages with Fern's library docs generator. | +| `make docs-fern-publish-staging` | Publish the Fern docs to the [staging instance](https://nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails). Only NeMo Guardrails maintainers can run this target. | +| `make docs-fern-publish-public` | Publish the Fern docs to the [public instance](https://nvidia-nemo-guardrails.docs.buildwithfern.com/nemo/guardrails), which is used for the [public documentation site](https://docs.nvidia.com/nemo/guardrails). Only NeMo Guardrails maintainers can run this target. | + +For pull requests that modify documentation, the docs build workflow checks the Fern docs and publishes a PR preview for same-repository branches. When a documentation pull request merges into `develop`, the workflow publishes the Fern docs to the [staging instance](https://nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails). + +Publishing the Fern docs to the public instance is a manual maintainer action after staging verification. The Fern CLI version is pinned in `fern/fern.config.json`. Do not run `fern upgrade` as part of normal documentation changes. @@ -157,15 +163,15 @@ To get started quickly, follow the steps below. 4. we use `Poetry` to manage the project dependencies. To install Poetry follow the instructions [here](https://python-poetry.org/docs/#installation): -> Note: This project requires Poetry version >=1.8,<2.0. Please ensure you are using a compatible version before running any Poetry commands. + > Note: This project requires Poetry version >=1.8,<2.0. Please ensure you are using a compatible version before running any Poetry commands. - Ensure you have `poetry` installed: + Ensure you have `poetry` installed: ```bash poetry --version ``` -6. Install the dev dependencies: +5. Install the dev dependencies: ```bash poetry install --with dev @@ -174,7 +180,7 @@ To get started quickly, follow the steps below. The preceding command installs pre-commit, pytest, and other development tools. Specify `--with dev,docs` to add the dependencies for building the documentation. -7. If needed, you can install extra dependencies as below: +6. If needed, you can install extra dependencies as below: ```bash poetry install --extras "openai tracing" @@ -183,13 +189,13 @@ To get started quickly, follow the steps below. ``` - to install all the extras: + to install all the extras: - ```bash - poetry install --all-extras - ``` + ```bash + poetry install --all-extras + ``` -> **Note**: `dev` is not part of the extras but it is an optional dependency group, so you need to install it as instructed above. + > **Note**: `dev` is not part of the extras but it is an optional dependency group, so you need to install it as instructed above. 7. Set up pre-commit hooks: @@ -358,7 +364,7 @@ To install a dependency using Poetry without adding it to the `pyproject.toml` f 1. **Activate the Poetry virtual environment**: - Run `poetry shell` to activate the virtual environment managed by Poetry. -> **Note**: If you don't want to activate the virtual environment, you can use `poetry run` to run commands within the virtual environment. + > **Note**: If you don't want to activate the virtual environment, you can use `poetry run` to run commands within the virtual environment. 2. **Install the package using `pip`**: - Once inside the virtual environment, you can use `pip` to install the package without affecting the `pyproject.toml`. For example: diff --git a/Makefile b/Makefile index 2f0b402c0d..82349f1f1b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: help .PHONY: test test-parallel test-serial test-benchmark test-watch test-coverage test-profile warm-fastembed-cache -.PHONY: docs-fern docs-fern-strict docs-fern-live docs-fern-preview-watch docs-fern-generate-sdk docs-fern-fix-empty-links docs-check-links docs-check-redirects +.PHONY: docs-fern docs-fern-strict docs-fern-live docs-fern-preview-watch docs-fern-generate-sdk docs-fern-fix-empty-links docs-check-links docs-check-redirects docs-fern-publish-staging docs-fern-publish-public .PHONY: pre-commit .DEFAULT_GOAL := help @@ -21,6 +21,7 @@ FASTEMBED_CACHE ?= .cache/fastembed FASTEMBED_MODEL ?= sentence-transformers/all-MiniLM-L6-v2 FASTEMBED_ENV ?= env FASTEMBED_CACHE_PATH=$(FASTEMBED_CACHE) FERN_STAGING_INSTANCE ?= nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails +FERN_PUBLIC_INSTANCE ?= nvidia-nemo-guardrails.docs.buildwithfern.com/nemo/guardrails test: $(UNIT_TEST_ENV) $(PYTEST) -n $(WORKERS) --dist $(DIST) $(ARGS) $(TEST) @@ -56,6 +57,9 @@ docs-fern-live: docs-fern-generate-sdk docs-fern-publish-staging: docs-fern-generate-sdk FERN_VERSION=$$(node -p "require('./fern/fern.config.json').version") && cd fern && npx --yes "fern-api@$${FERN_VERSION}" generate --docs --instance "$(FERN_STAGING_INSTANCE)" +docs-fern-publish-public: docs-fern-generate-sdk + FERN_VERSION=$$(node -p "require('./fern/fern.config.json').version") && cd fern && npx --yes "fern-api@$${FERN_VERSION}" generate --docs --instance "$(FERN_PUBLIC_INSTANCE)" + docs-fern-preview-watch: docs-fern-generate-sdk node scripts/watch-fern-preview.mjs @@ -101,6 +105,7 @@ help: ' docs-fern-strict Check Fern docs using the pinned Fern CLI' \ ' docs-fern-live Serve Fern docs locally' \ ' docs-fern-publish-staging Publish Fern docs to the staging instance' \ + ' docs-fern-publish-public Publish Fern docs to the public instance' \ ' docs-fern-preview-watch Watch and publish Fern preview for the current branch' \ ' docs-fern-generate-sdk Regenerate Python SDK reference pages with Fern' \ ' docs-fern-fix-empty-links Replace empty Markdown links with titles from Fern navigation' \ diff --git a/docs/LIVE_DOCS.mdx b/docs/LIVE_DOCS.mdx index 7ebcf6e7b5..f35c9ef8f5 100644 --- a/docs/LIVE_DOCS.mdx +++ b/docs/LIVE_DOCS.mdx @@ -53,6 +53,16 @@ make docs-fern-strict This regenerates the SDK reference, normalizes it, and runs `fern check`. +### CI Publishing + +The docs build workflow checks Fern docs on documentation pull requests and publishes a Fern preview for same-repository branches. When a documentation pull request merges into `develop`, the workflow publishes the Fern docs to the [staging instance](https://nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails). + +After staging verification, maintainers with NVIDIA Fern admin access can publish the staged docs to the [public instance](https://nvidia-nemo-guardrails.docs.buildwithfern.com/nemo/guardrails): + +```bash +make docs-fern-publish-public +``` + ### Direct Fern Command ```bash From 5d96c4b17870c6e68f81be0a5840b4760125137a Mon Sep 17 00:00:00 2001 From: Tim Gasser <200644301+tgasser-nv@users.noreply.github.com> Date: Fri, 12 Jun 2026 06:36:19 -0500 Subject: [PATCH 02/21] fix(docs): Skip Fern bash-script tests on Windows (#2017) --- tests/test_docs_links.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index 30c613955e..e9ea82fe44 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -14,11 +14,14 @@ # limitations under the License. import os +import platform import shutil import subprocess import tempfile from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] CHECK_DOCS_LINKS = REPO_ROOT / "scripts" / "check-docs-links.sh" @@ -38,6 +41,7 @@ def run_link_check(file_path: Path, env: dict[str, str] | None = None) -> subpro ) +@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") def test_reports_broken_local_markdown_links_with_source_line_numbers( tmp_path: Path, ) -> None: @@ -67,6 +71,7 @@ def test_reports_broken_local_markdown_links_with_source_line_numbers( assert "inside-code-fence.md" not in output +@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") def test_ignores_links_inside_inline_code_and_html_comments(tmp_path: Path) -> None: md_path = tmp_path / "guide.md" md_path.write_text( @@ -90,6 +95,7 @@ def test_ignores_links_inside_inline_code_and_html_comments(tmp_path: Path) -> N assert "inside-comment.md" not in output +@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") def test_resolves_guardrails_fern_routes(tmp_path: Path) -> None: md_path = tmp_path / "guide.mdx" md_path.write_text( @@ -112,6 +118,7 @@ def test_resolves_guardrails_fern_routes(tmp_path: Path) -> None: assert result.returncode == 0, output +@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") def test_rejects_mdx_suffixes_for_links_that_resolve_as_fern_routes() -> None: temp_dir = Path(tempfile.mkdtemp(prefix="check-docs-route-suffix-", dir=REPO_ROOT / "docs")) try: @@ -161,6 +168,7 @@ def test_rejects_mdx_suffixes_for_links_that_resolve_as_fern_routes() -> None: shutil.rmtree(temp_dir) +@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") def test_fails_loudly_on_malformed_html_comments(tmp_path: Path) -> None: md_path = tmp_path / "guide.md" md_path.write_text( From 342b5f5b0e424b10b234878e355482257dd341ec Mon Sep 17 00:00:00 2001 From: Tim Gasser <200644301+tgasser-nv@users.noreply.github.com> Date: Fri, 12 Jun 2026 07:05:49 -0500 Subject: [PATCH 03/21] feat(iorails): Telemetry - Add usage, request, response attributes to LLM span (#2009) --- nemoguardrails/guardrails/engine_registry.py | 75 +++++- nemoguardrails/guardrails/model_engine.py | 17 +- nemoguardrails/guardrails/telemetry.py | 120 +++++++++ nemoguardrails/tracing/constants.py | 8 + tests/guardrails/test_engine_registry.py | 269 ++++++++++++++++++- tests/guardrails/test_model_engine.py | 14 +- tests/guardrails/test_telemetry_spans.py | 156 +++++++++++ 7 files changed, 637 insertions(+), 22 deletions(-) diff --git a/nemoguardrails/guardrails/engine_registry.py b/nemoguardrails/guardrails/engine_registry.py index 6437896018..d68bb21d8f 100644 --- a/nemoguardrails/guardrails/engine_registry.py +++ b/nemoguardrails/guardrails/engine_registry.py @@ -33,6 +33,8 @@ api_call_span, llm_call_span, set_llm_call_content, + set_llm_request_attributes, + set_llm_response_attributes, ) from nemoguardrails.rails.llm.config import Model, RailsConfigData from nemoguardrails.tracing.constants import ( @@ -204,11 +206,22 @@ async def model_call(self, model_type: str, messages: list[dict], **kwargs: Any) else nullcontext() ) with llm_call_span(self._tracer, engine.model_name, provider_name, operation_name) as span: + # Request params are known before the call, so set them first — + # they land on the span even if the call raises. + set_llm_request_attributes(span, kwargs) with duration_ctx: result = await engine.chat_completion(messages, **kwargs) - # Capture content inside the span context so the helper sees - # the live LLM CLIENT span (not None even on the success path) - # and the attributes/events land before the span closes. + # Set response/usage and content attrs inside the span context so + # the helpers see the live LLM CLIENT span and the attributes land + # before it closes. Both are skipped on exception, which never + # reaches here. + set_llm_response_attributes( + span, + model=result.model, + response_id=result.request_id, + finish_reason=result.finish_reason, + usage=result.usage, + ) if self._content_capture_enabled: set_llm_call_content(span, messages, result.content) @@ -237,6 +250,14 @@ async def stream_model_call( observation is emitted on early consumer cancellation or on provider error mid-stream. + The LLM CLIENT span receives ``gen_ai.request.*`` attributes + (including ``gen_ai.request.stream=True``) before the first chunk, + and ``gen_ai.response.*`` / ``gen_ai.usage.*`` attributes + accumulated across the chunks after natural exhaustion. Like the + token metric, the response attrs are skipped on cancellation or a + mid-stream provider error. These span attrs are independent of + whether metrics are enabled. + Raises: KeyError: If no engine is registered with the given name. TypeError: If the named engine is not a ModelEngine. @@ -249,11 +270,17 @@ async def stream_model_call( provider_name = engine.model_config.engine or "unknown" operation_name = "chat" - # Capture the most recent chunk's ``usage`` field so we can emit - # token metrics after the stream completes — providers (e.g. - # OpenAI-compatible) only populate ``usage`` on the terminal - # chunk when ``stream_options.include_usage=true``. + # Capture the latest non-None response fields from the stream so we + # can set the LLM span's response/usage attrs and emit the token + # metric after the stream completes. Providers spread these across + # the SSE chunks — OpenAI-compatible engines only populate ``usage`` + # on the terminal chunk (when ``stream_options.include_usage=true``) + # and finish_reason likewise arrives last — so each field keeps its + # latest non-None value. captured_usage: Optional["UsageInfo"] = None + captured_model: Optional[str] = None + captured_response_id: Optional[str] = None + captured_finish_reason: Optional[str] = None # Accumulate streamed delta_content here when content capture is on; # joined and recorded onto the LLM span at stream end. The list is # allocated unconditionally (cost: one empty list per stream); the @@ -266,6 +293,9 @@ async def stream_model_call( else nullcontext() ) with llm_call_span(self._tracer, engine.model_name, provider_name, operation_name) as span: + # Set request params + stream=True before the first chunk so they + # land on the span even if the stream errors mid-flight. + set_llm_request_attributes(span, kwargs, stream=True) with duration_ctx: # Gate timing-state setup on ``_metrics_enabled`` so the # cold path skips ``time.monotonic()`` and the per-chunk @@ -290,16 +320,33 @@ async def stream_model_call( engine.model_name, provider_name, operation_name, now - last_chunk_time ) last_chunk_time = now - if chunk.usage is not None: - captured_usage = chunk.usage + # Keep the latest non-None response field from each chunk. + # Captured regardless of ``_metrics_enabled`` because they + # feed the span's response/usage attrs (set after the loop) + # as well as the token-usage metric. + if chunk.model is not None: + captured_model = chunk.model + if chunk.request_id is not None: + captured_response_id = chunk.request_id + if chunk.finish_reason is not None: + captured_finish_reason = chunk.finish_reason + if chunk.usage is not None: + captured_usage = chunk.usage if self._content_capture_enabled and chunk.delta_content: content_parts.append(chunk.delta_content) yield chunk - # Capture accumulated stream content inside the span context so - # the helper sees the live LLM CLIENT span before it closes. - # Reached only on natural exhaustion — consumer cancellation or - # provider error raises out of the ``with`` blocks above, in - # which case partial content is intentionally not recorded. + # Set response/usage attrs and (when enabled) content inside the + # span context so the helpers see the live LLM CLIENT span before + # it closes. Reached only on natural exhaustion — consumer + # cancellation or provider error raises out of the ``with`` blocks + # above, so partial response data is intentionally not recorded. + set_llm_response_attributes( + span, + model=captured_model, + response_id=captured_response_id, + finish_reason=captured_finish_reason, + usage=captured_usage, + ) # Empty ``content_parts`` -> output_text=None so we don't claim # an empty assistant response (matches iorails.py's request-span # streaming path). diff --git a/nemoguardrails/guardrails/model_engine.py b/nemoguardrails/guardrails/model_engine.py index 5bea39dc33..f962876f4d 100644 --- a/nemoguardrails/guardrails/model_engine.py +++ b/nemoguardrails/guardrails/model_engine.py @@ -117,9 +117,18 @@ def _parse_chat_completion_chunk(chunk: dict) -> Optional[LLMResponseChunk]: """Build an LLMResponseChunk from an SSE chunk dict. Returns None for chunks without one of: content delta, reasoning delta, - or a usage payload. - Role-only first events and finish-only events with empty deltas - map to None. + a usage payload, or a finish_reason. + Role-only first events map to None. + + Finish-only frames are preserved: a delta with no content/reasoning + (OpenAI sends ``delta: {}``, NIM sends ``delta: {"content": ""}``) and no + usage, carrying only a ``finish_reason``. Dropping them would strip + ``gen_ai.response.finish_reasons`` from the LLM span. (Some providers + instead attach ``finish_reason`` to the final content chunk — that case is + already captured, since content keeps the chunk alive.) When + ``stream_options.include_usage=true`` the usage payload arrives in a + *separate* later frame with empty ``choices`` — so finish_reason and usage + do not share a frame. Last chunk from OpenAI-compatible providers has a ``usage`` field when ``stream_options.include_usage=true``. This is passed through to capture @@ -138,7 +147,7 @@ def _parse_chat_completion_chunk(chunk: dict) -> Optional[LLMResponseChunk]: delta_reasoning = delta.get("reasoning_content") or None finish_reason = choice.get("finish_reason") - if not delta_content and not delta_reasoning and not usage_dict: + if not delta_content and not delta_reasoning and not usage_dict and not finish_reason: return None return LLMResponseChunk( diff --git a/nemoguardrails/guardrails/telemetry.py b/nemoguardrails/guardrails/telemetry.py index c60cb19900..16b22b422d 100644 --- a/nemoguardrails/guardrails/telemetry.py +++ b/nemoguardrails/guardrails/telemetry.py @@ -83,6 +83,7 @@ from nemoguardrails.guardrails.async_work_queue import AsyncWorkQueue from nemoguardrails.rails.llm.config import MetricsConfig, TracingConfig + from nemoguardrails.types import UsageInfo _OTEL_AVAILABLE = True else: @@ -576,6 +577,125 @@ def set_llm_call_content( _set_llm_call_content_events(span, input_messages, output_text) +# Maps an LLM request kwarg (as forwarded into the provider request body) +# to the OTEL GenAI span attribute that records it. Both ``max_tokens`` +# and the OpenAI ``max_completion_tokens`` alias map to the same attribute. +# ``stop`` / ``stop_sequences`` is handled separately by +# :func:`_stop_sequences` because it needs list normalization. +_GENAI_REQUEST_PARAMS = { + "temperature": GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE, + "max_tokens": GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS, + "max_completion_tokens": GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS, + "top_p": GenAIAttributes.GEN_AI_REQUEST_TOP_P, + "top_k": GenAIAttributes.GEN_AI_REQUEST_TOP_K, + "frequency_penalty": GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY, + "presence_penalty": GenAIAttributes.GEN_AI_REQUEST_PRESENCE_PENALTY, +} + + +def _stop_sequences(params: dict) -> Optional[list]: + """Return the request's stop sequences as a list, or ``None`` if unset. + + Reads the provider-specific request field that carries them: + + * OpenAI: ``stop`` + * Anthropic: ``stop_sequences`` + + A bare string is wrapped into a single-element list + (``gen_ai.request.stop_sequences`` is a string[]); a non-empty list is + returned unchanged. An empty or missing value, or any other type, + yields ``None`` — an empty ``stop`` is skipped rather than recorded as + a misleading empty span attribute. + """ + raw = params.get("stop") + if raw is None: + raw = params.get("stop_sequences") + if not raw: + return None + if isinstance(raw, str): + return [raw] + if isinstance(raw, list): + return raw + return None + + +def set_llm_request_attributes( + span: Optional["Span"], + params: dict, + *, + stream: bool = False, +) -> None: + """Set ``gen_ai.request.*`` attributes on an LLM CLIENT span. + + *params* is the kwargs dict forwarded to the model engine + (``GenerationOptions.llm_params``); only the known request-parameter + keys are mapped — any other kwargs are ignored. ``stop`` / + ``stop_sequences`` is normalized to a list via :func:`_stop_sequences`. + ``gen_ai.request.stream`` is set only when *stream* is True (omitted + otherwise, per the spec's conditionally-required-iff-streaming rule). + + These are non-sensitive sampling parameters (Recommended by spec), so + unlike message content they are recorded whenever the span exists — + there is no content-capture gate. Safe to call with ``span=None`` + (no-op) so callers don't have to branch on whether tracing is enabled. + """ + if span is None: + return + for key, attr in _GENAI_REQUEST_PARAMS.items(): + value = params.get(key) + if value is not None: + span.set_attribute(attr, value) + stop_sequences = _stop_sequences(params) + if stop_sequences is not None: + span.set_attribute(GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES, stop_sequences) + if stream: + span.set_attribute(GenAIAttributes.GEN_AI_REQUEST_STREAM, True) + + +def set_llm_response_attributes( + span: Optional["Span"], + *, + model: Optional[str] = None, + response_id: Optional[str] = None, + finish_reason: Optional[str] = None, + usage: Optional["UsageInfo"] = None, +) -> None: + """Set ``gen_ai.response.*`` and ``gen_ai.usage.*`` attrs on an LLM CLIENT span. + + Each attribute is set only when its source value is non-``None`` so + backends can distinguish an absent value from a real zero. + *finish_reason* is a single value wrapped into a one-element list to + match the spec's ``gen_ai.response.finish_reasons`` string[] shape. + Reasoning tokens are recorded only when the provider returned them. + ``gen_ai.usage.total_tokens`` is intentionally never emitted — it was + removed from the current spec. + + Callers feed the values from whatever source they have: the + non-streaming path reads them off the returned ``LLMResponse``; the + streaming path passes the fields accumulated across chunks (model and + id arrive early, finish_reason and usage on the terminal chunk). Like + :func:`set_llm_request_attributes`, these are non-sensitive telemetry + recorded whenever the span exists — no content-capture gate. Safe to + call with ``span=None`` (no-op). + """ + if span is None: + return + if model is not None: + span.set_attribute(GenAIAttributes.GEN_AI_RESPONSE_MODEL, model) + if response_id is not None: + span.set_attribute(GenAIAttributes.GEN_AI_RESPONSE_ID, response_id) + if finish_reason is not None: + span.set_attribute(GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) + if usage is not None: + span.set_attribute(GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens) + span.set_attribute(GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, usage.output_tokens) + if usage.reasoning_tokens is not None: + span.set_attribute( + GenAIAttributes.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + usage.reasoning_tokens, + ) + + def set_request_content( span: Optional["Span"], input_messages: LLMMessages, diff --git a/nemoguardrails/tracing/constants.py b/nemoguardrails/tracing/constants.py index 563dfb7e1d..2cc954638a 100644 --- a/nemoguardrails/tracing/constants.py +++ b/nemoguardrails/tracing/constants.py @@ -130,6 +130,10 @@ class GenAIAttributes: GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" GEN_AI_REQUEST_STOP_SEQUENCES = "gen_ai.request.stop_sequences" + # Conditionally Required per spec: set if and only if the request is + # streaming (omit entirely on non-streaming calls). + GEN_AI_REQUEST_STREAM = "gen_ai.request.stream" + GEN_AI_RESPONSE_MODEL = "gen_ai.response.model" GEN_AI_RESPONSE_ID = "gen_ai.response.id" GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons" @@ -138,6 +142,10 @@ class GenAIAttributes: GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens" + # Recommended span attribute (when applicable, e.g. reasoning models). + # Span-only — NOT a valid ``gen_ai.token.type`` metric label value. + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning.output_tokens" + # Required label on the ``gen_ai.client.token.usage`` metric. # Allowed values (from spec): "input" or "output" only. Reasoning # and cached tokens are span-only attributes, NOT valid token.type diff --git a/tests/guardrails/test_engine_registry.py b/tests/guardrails/test_engine_registry.py index 9bc569afb5..6397e25200 100644 --- a/tests/guardrails/test_engine_registry.py +++ b/tests/guardrails/test_engine_registry.py @@ -15,12 +15,16 @@ """Unit tests for engine_registry module.""" +import json from typing import Optional -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from nemoguardrails.guardrails import telemetry from nemoguardrails.guardrails.api_engine import APIEngine @@ -86,6 +90,28 @@ def manager_with_metrics(rails_config): return EngineRegistry(rails_config.models, rails_config.rails.config, metrics_enabled=True) +@pytest.fixture +def span_exporter(): + """Install a test-local TracerProvider + in-memory exporter and return + ``(tracer, exporter)``. The tracer is passed explicitly to the registry + (no global TracerProvider is set), so there is no global state to clean + up beyond the autouse ``reset_telemetry_singletons`` fixture.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = provider.get_tracer("test") + return tracer, exporter + + +@pytest.fixture +@patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) +def manager_with_tracer(rails_config, span_exporter): + """Create an EngineRegistry wired to the test tracer (metrics + content + capture off) so LLM calls produce real spans we can read back.""" + tracer, _ = span_exporter + return EngineRegistry(rails_config.models, rails_config.rails.config, tracer=tracer) + + def _mock_stream(*chunks: LLMResponseChunk, error: Optional[Exception] = None): """Build an async generator that yields ``chunks`` in order, then optionally raises ``error``. Drop-in replacement for inline @@ -102,6 +128,32 @@ async def _gen(msgs, **kwargs): # noqa: ARG001 (signature dictated by ModelEngi return _gen +def _mock_sse_response(raw_chunks: list[dict]): + """Build a mock aiohttp streaming response that emits ``raw_chunks`` as + SSE ``data:`` frames followed by ``[DONE]``. + + Drives ModelEngine.stream_call's real ``_parse_chat_completion_chunk`` + path (rather than ``_mock_stream``'s pre-parsed chunks), so a finish-only + SSE frame is parsed end-to-end. readline() returns one ``\\n``-terminated + line at a time, matching aiohttp's StreamReader. + """ + lines = [f"data: {json.dumps(chunk)}\n".encode() for chunk in raw_chunks] + lines.append(b"data: [DONE]\n") + line_iter = iter(lines) + + async def _readline(): + return next(line_iter, b"") + + mock_content = MagicMock() + mock_content.readline = _readline + + mock_response = AsyncMock() + mock_response.__aenter__ = AsyncMock(return_value=mock_response) + mock_response.status = 200 + mock_response.content = mock_content + return mock_response + + class TestEngineRegistryInit: """Test EngineRegistry creates engines from config.""" @@ -370,6 +422,89 @@ async def test_no_metrics_emitted_when_metrics_disabled(self, manager, metric_re assert "gen_ai.client.operation.duration" not in points +class TestEngineRegistryModelCallSpanAttributes: + """``model_call`` sets gen_ai.request.* and gen_ai.response.* / usage.* + attributes on the LLM CLIENT span, independent of metrics and content + capture.""" + + @pytest.mark.asyncio + async def test_sets_request_and_response_attributes(self, manager_with_tracer, span_exporter): + """Populated LLMResponse + request kwargs → the finished span carries + usage, response, and request-param attrs; gen_ai.request.stream is + absent on the non-streaming path and total_tokens is never emitted.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock( + return_value=LLMResponse( + content="hi there", + model="meta/llama-3.3-70b-instruct", + finish_reason="stop", + request_id="chatcmpl-xyz", + usage=UsageInfo(input_tokens=10, output_tokens=5, total_tokens=15, reasoning_tokens=3), + ), + ) + + await manager_with_tracer.model_call( + "main", + [{"role": "user", "content": "hi"}], + temperature=0.5, + max_tokens=100, + stop=["END"], + ) + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.request.temperature"] == 0.5 + assert attrs["gen_ai.request.max_tokens"] == 100 + assert list(attrs["gen_ai.request.stop_sequences"]) == ["END"] + assert "gen_ai.request.stream" not in attrs + assert attrs["gen_ai.response.model"] == "meta/llama-3.3-70b-instruct" + assert attrs["gen_ai.response.id"] == "chatcmpl-xyz" + assert list(attrs["gen_ai.response.finish_reasons"]) == ["stop"] + assert attrs["gen_ai.usage.input_tokens"] == 10 + assert attrs["gen_ai.usage.output_tokens"] == 5 + assert attrs["gen_ai.usage.reasoning.output_tokens"] == 3 + assert "gen_ai.usage.total_tokens" not in attrs + + @pytest.mark.asyncio + async def test_attributes_set_without_metrics_or_content_capture(self, manager_with_tracer, span_exporter): + """The new attrs are independent of metrics and content capture: with + both off (the manager_with_tracer default), usage/response attrs are + still present while message-content attrs are not.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock( + return_value=LLMResponse(content="hi", usage=UsageInfo(input_tokens=2, output_tokens=1)), + ) + + await manager_with_tracer.model_call("main", [{"role": "user", "content": "hi"}]) + + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.usage.input_tokens"] == 2 + assert attrs["gen_ai.usage.output_tokens"] == 1 + assert "gen_ai.input.messages" not in attrs + assert "guardrails.request.input" not in attrs + + @pytest.mark.asyncio + async def test_request_attributes_present_on_error(self, manager_with_tracer, span_exporter): + """Request params are set before the call, so they survive on the span + when the call raises; response/usage attrs are absent and the span is + marked ERROR via error.type.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock(side_effect=RuntimeError("provider down")) + + with pytest.raises(RuntimeError, match="provider down"): + await manager_with_tracer.model_call("main", [{"role": "user", "content": "hi"}], temperature=0.2) + + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == 0.2 + assert "gen_ai.usage.input_tokens" not in attrs + assert "gen_ai.response.model" not in attrs + assert attrs["error.type"] == "RuntimeError" + + class TestEngineRegistryStartErrors: """Test EngineRegistry start() error handling and rollback.""" @@ -1028,3 +1163,135 @@ async def test_provider_error_after_first_chunk_records_partial_timing(self, man assert points["gen_ai.client.operation.time_to_first_chunk"][0].value == 1 assert "gen_ai.client.operation.time_per_output_chunk" not in points assert points["gen_ai.client.operation.duration"][0].attributes["error.type"] == "RuntimeError" + + +class TestEngineRegistryStreamModelCallSpanAttributes: + """``stream_model_call`` sets gen_ai.request.* (including stream=True) and + the accumulated gen_ai.response.* / usage.* attributes on the LLM CLIENT + span, independent of metrics and content capture (both off here).""" + + @pytest.mark.asyncio + async def test_accumulates_response_attributes_across_chunks(self, manager_with_tracer, span_exporter): + """Response fields arrive on different chunks (model + id early, + finish_reason + usage on the terminal chunk); the span carries the + accumulated values plus the request params and stream=True.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.stream_chat_completion = _mock_stream( + LLMResponseChunk( + delta_content="Hello", + model="meta/llama-3.3-70b-instruct", + request_id="chatcmpl-stream", + ), + LLMResponseChunk(delta_content=" world"), + LLMResponseChunk( + finish_reason="stop", + usage=UsageInfo(input_tokens=8, output_tokens=4, total_tokens=12, reasoning_tokens=2), + ), + ) + + async for _ in manager_with_tracer.stream_model_call( + "main", [{"role": "user", "content": "hi"}], temperature=0.3, stop=["X"] + ): + pass + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attrs = dict(spans[0].attributes) + assert attrs["gen_ai.request.stream"] is True + assert attrs["gen_ai.request.temperature"] == 0.3 + assert list(attrs["gen_ai.request.stop_sequences"]) == ["X"] + assert attrs["gen_ai.response.model"] == "meta/llama-3.3-70b-instruct" + assert attrs["gen_ai.response.id"] == "chatcmpl-stream" + assert list(attrs["gen_ai.response.finish_reasons"]) == ["stop"] + assert attrs["gen_ai.usage.input_tokens"] == 8 + assert attrs["gen_ai.usage.output_tokens"] == 4 + assert attrs["gen_ai.usage.reasoning.output_tokens"] == 2 + assert "gen_ai.usage.total_tokens" not in attrs + + @pytest.mark.asyncio + async def test_finish_only_sse_frame_lands_finish_reasons_on_span(self, manager_with_tracer, span_exporter): + """End-to-end regression for the dropped finish-only frame. A real + OpenAI-style stream delivers ``finish_reason`` in a frame with an empty + delta and no usage, then usage in a separate empty-``choices`` frame. + Driving the actual ``_parse_chat_completion_chunk`` (not a pre-parsed + ``_mock_stream``), the span must still carry + ``gen_ai.response.finish_reasons`` — restoring the ``is None``-only + parser guard would drop the finish frame and fail this assertion.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine._client = AsyncMock() + engine._client.post = MagicMock( + return_value=_mock_sse_response( + [ + { + "id": "chatcmpl-stream", + "model": "meta/llama-3.3-70b-instruct", + "choices": [{"delta": {"content": "Hello"}, "finish_reason": None}], + }, + {"choices": [{"delta": {"content": " world"}, "finish_reason": None}]}, + # Finish-only frame: empty delta, no usage — previously dropped. + {"choices": [{"delta": {}, "finish_reason": "stop"}]}, + # Usage arrives on a separate empty-choices frame. + {"choices": [], "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}}, + ] + ) + ) + engine._running = True + + async for _ in manager_with_tracer.stream_model_call("main", [{"role": "user", "content": "hi"}]): + pass + + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert list(attrs["gen_ai.response.finish_reasons"]) == ["stop"] + assert attrs["gen_ai.response.model"] == "meta/llama-3.3-70b-instruct" + assert attrs["gen_ai.response.id"] == "chatcmpl-stream" + assert attrs["gen_ai.usage.input_tokens"] == 8 + assert attrs["gen_ai.usage.output_tokens"] == 4 + assert attrs["gen_ai.request.stream"] is True + + @pytest.mark.asyncio + async def test_stream_attribute_set_even_without_usage(self, manager_with_tracer, span_exporter): + """gen_ai.request.stream=True is set before the first chunk, so it is + present even when no chunk carries usage; usage attrs are then absent.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.stream_chat_completion = _mock_stream( + LLMResponseChunk(delta_content="Hello"), + LLMResponseChunk(delta_content=" world"), + ) + + async for _ in manager_with_tracer.stream_model_call("main", [{"role": "user", "content": "hi"}]): + pass + + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.stream"] is True + assert "gen_ai.usage.input_tokens" not in attrs + assert "gen_ai.usage.output_tokens" not in attrs + + @pytest.mark.asyncio + async def test_request_attributes_present_on_provider_error(self, manager_with_tracer, span_exporter): + """Provider errors mid-stream → request attrs (incl. stream) survive on + the span; the post-loop response/usage attrs are never set (even though + a chunk carried ``model``), and the span is marked ERROR.""" + _, exporter = span_exporter + engine = manager_with_tracer._get_engine("main", ModelEngine) + engine.stream_chat_completion = _mock_stream( + LLMResponseChunk(delta_content="Hello", model="meta/llama-3.3-70b-instruct"), + error=RuntimeError("provider died"), + ) + + with pytest.raises(RuntimeError, match="provider died"): + async for _ in manager_with_tracer.stream_model_call( + "main", [{"role": "user", "content": "hi"}], temperature=0.9 + ): + pass + + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.stream"] is True + assert attrs["gen_ai.request.temperature"] == 0.9 + # Captured during iteration but not written — response attrs are set + # only after natural exhaustion, which the error skips. + assert "gen_ai.response.model" not in attrs + assert "gen_ai.usage.input_tokens" not in attrs + assert attrs["error.type"] == "RuntimeError" diff --git a/tests/guardrails/test_model_engine.py b/tests/guardrails/test_model_engine.py index 8c6354ea90..261a8e0132 100644 --- a/tests/guardrails/test_model_engine.py +++ b/tests/guardrails/test_model_engine.py @@ -1306,9 +1306,17 @@ def test_content_chunk_with_no_usage_has_usage_none(self): assert result is not None assert result.usage is None - def test_finish_only_delta_returns_none(self): - """Finish-only deltas (no content/reasoning) are skipped, matching prior behavior.""" - assert _parse_chat_completion_chunk({"choices": [{"delta": {}, "finish_reason": "stop"}]}) is None + def test_finish_only_delta_returns_chunk_with_finish_reason(self): + """Finish-only frames (empty delta, no usage) are preserved so the + ``finish_reason`` reaches the LLM span's ``gen_ai.response.finish_reasons``. + Real OpenAI-compatible providers deliver ``finish_reason`` in a terminal + frame with an empty delta and no usage.""" + result = _parse_chat_completion_chunk({"choices": [{"delta": {}, "finish_reason": "stop"}]}) + assert result is not None + assert result.delta_content is None + assert result.delta_reasoning is None + assert result.usage is None + assert result.finish_reason == "stop" def test_passes_through_metadata(self): """model, request id, and finish_reason flow into the chunk when content is present.""" diff --git a/tests/guardrails/test_telemetry_spans.py b/tests/guardrails/test_telemetry_spans.py index 6a262fc7ea..6fcd28fae1 100644 --- a/tests/guardrails/test_telemetry_spans.py +++ b/tests/guardrails/test_telemetry_spans.py @@ -29,7 +29,10 @@ api_call_span, llm_call_span, rail_span, + set_llm_request_attributes, + set_llm_response_attributes, ) +from nemoguardrails.types import UsageInfo @pytest.fixture @@ -332,3 +335,156 @@ def test_records_error_type_on_generator_exit(self, otel_provider): def test_noop_when_tracer_none(self): with api_call_span(None, "jailbreak_detection") as span: assert span is None + + +def _span_attrs(otel_provider, set_fn): + """Run ``set_fn(span)`` inside a finished CLIENT span and return its + attributes as a plain dict. + + Mirrors how the helpers are used in production (called on a live span + inside the ``llm_call_span`` block) and reads the result back off the + exported span, the same way the span-helper tests above do. + """ + provider, exporter = otel_provider + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("chat test-model") as span: + set_fn(span) + return dict(exporter.get_finished_spans()[-1].attributes) + + +class TestSetLlmRequestAttributes: + def test_maps_scalar_params(self, otel_provider): + """Every supported scalar sampling param maps to its gen_ai.request.* attr.""" + params = { + "temperature": 0.7, + "max_tokens": 256, + "top_p": 0.9, + "top_k": 40, + "frequency_penalty": 0.5, + "presence_penalty": 0.25, + } + attrs = _span_attrs(otel_provider, lambda s: set_llm_request_attributes(s, params)) + assert attrs["gen_ai.request.temperature"] == 0.7 + assert attrs["gen_ai.request.max_tokens"] == 256 + assert attrs["gen_ai.request.top_p"] == 0.9 + assert attrs["gen_ai.request.top_k"] == 40 + assert attrs["gen_ai.request.frequency_penalty"] == 0.5 + assert attrs["gen_ai.request.presence_penalty"] == 0.25 + + def test_max_completion_tokens_aliases_max_tokens(self, otel_provider): + """The OpenAI ``max_completion_tokens`` alias lands on gen_ai.request.max_tokens.""" + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_request_attributes(s, {"max_completion_tokens": 128}), + ) + assert attrs["gen_ai.request.max_tokens"] == 128 + + @pytest.mark.parametrize( + "params, expected", + [ + ({"stop": "END"}, ["END"]), + ({"stop": ["a", "b"]}, ["a", "b"]), + ({"stop_sequences": ["x"]}, ["x"]), + ({"stop": 123}, None), # malformed type → skipped + ({"stop": []}, None), # empty list → skipped + ({"stop": ""}, None), # empty string → skipped + ], + ids=["string", "list", "stop_sequences_key", "malformed", "empty_list", "empty_string"], + ) + def test_stop_sequences_normalization(self, otel_provider, params, expected): + """``stop`` / ``stop_sequences`` normalize to gen_ai.request.stop_sequences: + a string wraps to a one-element list, a list passes through, and + empty/malformed values are skipped entirely (no empty attribute, which + would falsely imply stop tokens were configured).""" + attrs = _span_attrs(otel_provider, lambda s: set_llm_request_attributes(s, params)) + if expected is None: + assert "gen_ai.request.stop_sequences" not in attrs + else: + assert list(attrs["gen_ai.request.stop_sequences"]) == expected + + def test_unknown_kwargs_ignored(self, otel_provider): + """Kwargs with no gen_ai.request.* mapping are silently ignored.""" + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_request_attributes(s, {"temperature": 0.1, "foo": "bar"}), + ) + assert attrs["gen_ai.request.temperature"] == 0.1 + assert "foo" not in attrs + + def test_stream_true_sets_attribute(self, otel_provider): + """``stream=True`` records gen_ai.request.stream (CR iff streaming).""" + attrs = _span_attrs(otel_provider, lambda s: set_llm_request_attributes(s, {}, stream=True)) + assert attrs["gen_ai.request.stream"] is True + + def test_stream_default_omits_attribute(self, otel_provider): + """The default (non-streaming) call omits gen_ai.request.stream entirely.""" + attrs = _span_attrs(otel_provider, lambda s: set_llm_request_attributes(s, {})) + assert "gen_ai.request.stream" not in attrs + + def test_noop_when_span_none(self): + """``span=None`` is a no-op and must not raise.""" + set_llm_request_attributes(None, {"temperature": 0.5}, stream=True) + + +class TestSetLlmResponseAttributes: + def test_sets_all_response_and_usage_attributes(self, otel_provider): + """A fully-populated response sets every response + usage attr, including + reasoning tokens, and never emits the spec-removed total_tokens.""" + usage = UsageInfo(input_tokens=12, output_tokens=34, total_tokens=46, reasoning_tokens=7) + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_response_attributes( + s, + model="meta/llama-3.3-70b-instruct", + response_id="chatcmpl-abc123", + finish_reason="stop", + usage=usage, + ), + ) + assert attrs["gen_ai.response.model"] == "meta/llama-3.3-70b-instruct" + assert attrs["gen_ai.response.id"] == "chatcmpl-abc123" + assert list(attrs["gen_ai.response.finish_reasons"]) == ["stop"] + assert attrs["gen_ai.usage.input_tokens"] == 12 + assert attrs["gen_ai.usage.output_tokens"] == 34 + assert attrs["gen_ai.usage.reasoning.output_tokens"] == 7 + assert "gen_ai.usage.total_tokens" not in attrs + + def test_finish_reason_wrapped_in_list(self, otel_provider): + """The single finish_reason is wrapped into the spec's finish_reasons string[].""" + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_response_attributes(s, finish_reason="length"), + ) + assert list(attrs["gen_ai.response.finish_reasons"]) == ["length"] + + def test_reasoning_tokens_omitted_when_none(self, otel_provider): + """Reasoning tokens are recorded only when present; input/output still set.""" + usage = UsageInfo(input_tokens=1, output_tokens=2, reasoning_tokens=None) + attrs = _span_attrs(otel_provider, lambda s: set_llm_response_attributes(s, usage=usage)) + assert "gen_ai.usage.reasoning.output_tokens" not in attrs + assert attrs["gen_ai.usage.input_tokens"] == 1 + assert attrs["gen_ai.usage.output_tokens"] == 2 + + def test_usage_none_sets_no_usage_attributes(self, otel_provider): + """``usage=None`` records no usage attrs; non-usage fields still set.""" + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_response_attributes(s, model="m", usage=None), + ) + assert attrs["gen_ai.response.model"] == "m" + assert "gen_ai.usage.input_tokens" not in attrs + assert "gen_ai.usage.output_tokens" not in attrs + + def test_omits_none_response_fields(self, otel_provider): + """Each response field is omitted when its source value is None.""" + attrs = _span_attrs( + otel_provider, + lambda s: set_llm_response_attributes(s, model="only-model"), + ) + assert attrs["gen_ai.response.model"] == "only-model" + assert "gen_ai.response.id" not in attrs + assert "gen_ai.response.finish_reasons" not in attrs + + def test_noop_when_span_none(self): + """``span=None`` is a no-op and must not raise.""" + set_llm_response_attributes(None, model="m", usage=UsageInfo(input_tokens=1, output_tokens=2)) From 90ee1ddcb189072f5c9f5f1a4800629ab0c87864 Mon Sep 17 00:00:00 2001 From: Pouyan <13303554+Pouyanpi@users.noreply.github.com> Date: Fri, 12 Jun 2026 18:30:48 +0200 Subject: [PATCH 04/21] docs: add AI-assisted contribution policy and coding-agent guide (#2007) --- .github/ISSUE_TEMPLATE/refactor_proposal.yml | 70 +++ .github/PULL-REQUEST-TEMPLATE.md | 22 +- AGENTS.md | 171 ++++++ AI_POLICY.md | 39 ++ CLAUDE.md | 1 + CONTRIBUTING.md | 596 +++++-------------- nemoguardrails/AGENTS.md | 122 ++++ nemoguardrails/CLAUDE.md | 1 + pytest.ini | 1 - 9 files changed, 585 insertions(+), 438 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/refactor_proposal.yml create mode 100644 AGENTS.md create mode 100644 AI_POLICY.md create mode 120000 CLAUDE.md create mode 100644 nemoguardrails/AGENTS.md create mode 120000 nemoguardrails/CLAUDE.md diff --git a/.github/ISSUE_TEMPLATE/refactor_proposal.yml b/.github/ISSUE_TEMPLATE/refactor_proposal.yml new file mode 100644 index 0000000000..cc4402858f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/refactor_proposal.yml @@ -0,0 +1,70 @@ +name: Refactor Proposal +description: Propose a maintainer-led refactor for discussion +title: "refactor: " +labels: ["refactoring", "status: needs triage"] +body: + - type: markdown + attributes: + value: | + Refactors are maintainer-led and are not accepted as unsolicited pull requests. + + Use this form to explain the problem and proposed direction. Maintainers decide whether the refactor should happen, what plan is acceptable, and who, if anyone, should be assigned to implement it. + - type: checkboxes + attributes: + label: Required checks + description: Confirm before submitting this proposal. + options: + - label: I searched existing issues and pull requests for related refactor proposals. + required: true + - label: I understand that opening this issue does not mean a refactor PR will be accepted. + required: true + - label: I understand that a refactor PR should not be opened unless a maintainer approves the plan and assigns the work. + required: true + - type: textarea + attributes: + label: Problem + description: What is hard to maintain, unsafe, confusing, or blocking today? + placeholder: Describe the current design problem and why it matters. + validations: + required: true + - type: textarea + attributes: + label: Proposed direction + description: What structural direction do you think maintainers should consider? + placeholder: Keep this high level unless maintainers have asked for detailed design. + validations: + required: true + - type: textarea + attributes: + label: Migration and compatibility notes + description: What public APIs, config formats, docs, or user workflows could be affected? + validations: + required: true + - type: textarea + attributes: + label: Validation plan + description: What tests, docs checks, or compatibility checks would be needed? + validations: + required: true + - type: textarea + attributes: + label: Risks + description: List known risks, open questions, and possible alternatives. + validations: + required: true + - type: textarea + attributes: + label: Relevant branch or files + description: Optional. Share a WIP branch, affected files, or investigation notes if useful. + validations: + required: false + - type: dropdown + attributes: + label: Are you interested in helping after maintainer approval? + description: This does not assign you to the work. Maintainers decide ownership after triage. + options: + - I am only proposing this for maintainer consideration. + - I can help with investigation if maintainers ask. + - I can help implement after maintainers approve a plan and assign the work. + validations: + required: true diff --git a/.github/PULL-REQUEST-TEMPLATE.md b/.github/PULL-REQUEST-TEMPLATE.md index d451df3f91..21def034d5 100644 --- a/.github/PULL-REQUEST-TEMPLATE.md +++ b/.github/PULL-REQUEST-TEMPLATE.md @@ -1,18 +1,36 @@ ## Description + why we should accept this pull request. Include any areas that need careful + review. --> ## Related Issue(s) +## Verification + + + +## AI Assistance + +- [ ] No AI tools were used. +- [ ] AI tools were used; a human reviewed and can explain every change (tool: ___). + ## Checklist - [ ] I've read the [CONTRIBUTING](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/CONTRIBUTING.md) guidelines. +- [ ] This PR links to a triaged issue assigned to me. +- [ ] My PR title follows the project commit convention. - [ ] I've updated the documentation if applicable. - [ ] I've added tests if applicable. +- [ ] I've noted any verification beyond CI and any checks I couldn't run. +- [ ] I did not update generated changelog files manually. +- [ ] I addressed all CodeRabbit, Greptile, and other review comments, or replied with why no change is needed. - [ ] @mentions of the person or team responsible for reviewing proposed changes. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..bdbaaa51ec --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,171 @@ +# AGENTS.md + +`CONTRIBUTING.md` is canonical for public contribution workflow: issues, +assignment, pull requests, refactors, changelogs, validation, commits, and DCO. +`AI_POLICY.md` is canonical for public AI-assisted contribution policy. + +When working inside `nemoguardrails/`, also follow `nemoguardrails/AGENTS.md` for +runtime, public-API, and provider-integration rules. + +## Quick Rules + +- Agent-specific rule: do not submit issues, PRs, or draft PRs through browser + automation, the GitHub API, `gh`, or similar tooling. Draft text for a human to + review and submit, following the repo's issue/PR templates so it can be + submitted as-is. +- Do not push branches or prepare public-submission-ready PR materials unless + the linked issue is triaged and assigned to the human contributor. +- Do not implement refactors unless a maintainer has approved the plan and + assigned the work. +- Never edit `CHANGELOG.md` or `CHANGELOG-Colang.md` manually. +- Do not commit secrets, credentials, or sensitive provider data, and do not + fabricate results, approvals, or citations. See `AI_POLICY.md` Safety and + Privacy for the canonical list. +- Do not add generated media, large generated assets, or synthetic datasets + without clear provenance and maintainer alignment. +- Unit tests must not call live LLM or provider services. +- Do not add license headers manually. Pre-commit handles license insertion. +- Do not add comments unless explicitly requested; keep existing comments, + docstrings, and license headers unless your change makes them inaccurate. +- Use Poetry for Python commands: `poetry run python ...`, + `poetry run pytest ...`, `poetry run pre-commit ...`. + +## Repository Map + +- Main package: `nemoguardrails/` +- Tests: `tests/` and `benchmark/tests` (the `testpaths` in `pytest.ini`) +- Schemas and validation snapshots: `schemas/` +- Default development branch: `develop` + +## Setup + +- Install development dependencies: + + ```bash + poetry install --with dev + ``` + +- Install documentation dependencies when working on docs: + + ```bash + poetry install --with dev,docs + ``` + +- Do not add dependencies to `pyproject.toml` or update `poetry.lock` unless the + task requires it. For temporary local investigation, use: + + ```bash + poetry run pip install + ``` + +- When a dependency change is required, keep it in the narrowest appropriate + dependency group or optional extra, add clear compatibility bounds, and avoid + moving optional integration dependencies into the default install path. + +## Validation + +Canonical command reference: `CONTRIBUTING.md` Validation. The table below adds +agent-operational diagnosis commands; see `CONTRIBUTING.md` for shared rows such +as tox and package coverage. + +| Task | Command | +| --- | --- | +| Run the test suite | `make test` (pytest-xdist parallel; unsets live-provider keys so unit tests cannot reach live services; runs all `pytest.ini` testpaths) | +| Focused test | `make test TEST=path/to/test_file.py::test_name` (extra flags via `ARGS="-k ... -q"`) | +| Serial, deterministic run | `make test WORKERS=1` (no parallelism, still unsets live keys) | +| Coverage | `make test-coverage` | +| Pre-commit hooks | `poetry run pre-commit run --all-files` | +| Docs check | `make docs-fern` | +| Ruff diagnosis | `poetry run ruff check path/to/file.py` | +| Ruff formatting diagnosis | `poetry run ruff format path/to/file.py` | +| Pyright diagnosis | `poetry run pyright` | + +| Change type | Minimum validation | +| --- | --- | +| Docs or repository metadata only | `poetry run pre-commit run --files `; build docs when rendering, links, examples, or docs configuration may be affected | +| Runtime bug fix | Focused regression test plus pre-commit on changed files; broaden when shared behavior is touched | +| Public API, config, or Colang behavior | Focused tests plus related docs/examples; add broader package tests when compatibility risk is meaningful | +| Server, streaming, tracing, actions, or generation | Targeted tests for the changed path and fallback/unsupported path | +| Packaging, dependencies, or lockfiles | Relevant install/package checks plus pre-commit; keep dependency diffs separate from unrelated changes | + +- For PR-ready code changes, pre-commit is the authoritative lint, format, + license-header, and type-checking path. +- Standalone Ruff, Ruff format, and Pyright runs are local diagnosis only; run + pre-commit on changed files before handoff and report if it is skipped. +- `make test` runs every `pytest.ini` testpath, so it includes `benchmark/tests`, + not just `tests/`; scope with `TEST=`. The default suite needs no network: + `tests/conftest.py` swaps the default FastEmbed model for a deterministic + provider, so only `real_embeddings`-marked tests use the real model. +- Diagnose isolation flakiness by comparing `make test` with `make test + WORKERS=1` (same env-safety, no parallelism); `serial`/`slow` markers in + `pytest.ini` are advisory and not enforced by the parallel runner. +- `make test-serial` and bare `poetry run pytest` do NOT unset live-provider + keys; prefer `make test` / `make test WORKERS=1` so unit tests cannot reach + live services. + +## Contribution Workflow + +- Follow `CONTRIBUTING.md` for issue, assignment, PR title, refactor, changelog, + validation, commit, and DCO policy. +- Follow `AI_POLICY.md` for disclosure, human accountability, safety, and + privacy requirements. +- For non-trivial features, API changes, refactors, or behavior changes without + clear maintainer direction (a linked issue that is triaged, assigned to you, + and has an agreed approach recorded in the thread), stop at a proposal or + implementation plan (an issue comment, or a throwaway `PLAN.md` PR maintainers + can review) rather than implementing. +- Before preparing PR-shaped work, check for duplicate or in-flight effort with + read-only `gh` (distinct from the no-`gh`-submission rule above): + `gh issue view --comments`, `gh pr list --state open --search " + in:body"`, and `gh pr list --state open --search ""`. If an open + PR already covers the change, do not prepare a duplicate; if your approach + differs materially, surface that difference in the issue draft for a maintainer. +- If work is exploratory, draft an issue comment with the branch and relevant + files instead of opening a PR. +- Use the Conventional Commit-style titles described in `CONTRIBUTING.md`. +- Do not prefix PR titles or commit messages with agent markers, and do not add + AI tools or agents as commit co-authors (no `Co-Authored-By` trailers for AI). + +## Review Readiness + +- Follow `CONTRIBUTING.md` for review-readiness policy (CodeRabbit, Greptile, + human comments, readiness labels): address or reply to every open review comment + before requesting maintainer review, do not resolve reviewers' own threads, and + do not self-apply the readiness label. + +## Code Changes + +- For maintainer-approved refactors, add or update characterization tests before + changing subtle behavior; when the existing suite does not cover the refactored + code, keep the equivalence check you used to prove behavior is unchanged. + +## Documentation And Generated Files + +- Update docs when changing user-visible behavior, public APIs, configuration + syntax, examples, or installation requirements. +- For optional integrations, document whether the integration is optional, which + extras or packages are required, which API keys or environment variables are + expected, and whether the integration uses the default OpenAI-compatible + framework path or the LangChain framework. +- When documenting model or provider examples, state the relevant model type, + routing mode, supported modes, and known limitations rather than assuming the + example generalizes to every backend. +- Use current generally-available model IDs in docs/examples (verify against the + provider's docs), and do not change shipped default model parameters as a + documentation update. +- Do not hand-edit generated files or lockfiles unless the task explicitly + requires regenerating them with project tooling. +- Put release-note context in issue or PR draft text instead of changelog files. +- For notebook documentation, follow `CONTRIBUTING.md`. Do not run + `build_notebook_docs.py` unless explicitly asked; it currently runs broad git + staging and pre-commit commands. Use a clean worktree if it must be run. + +## Review Mode + +- When reviewing a branch, compare against the merge base with `develop` and + inspect tests as well as implementation. +- Before handing off, run pre-commit first (per Validation, so line locations are + stable), then a structured code review (for example `codex review` or + `/code-review`) with a security pass for auth, input-handling, deserialization, + or external-call changes. Treat findings as advisory: verify each against the + real code path and loop until clean. diff --git a/AI_POLICY.md b/AI_POLICY.md new file mode 100644 index 0000000000..47a3471c17 --- /dev/null +++ b/AI_POLICY.md @@ -0,0 +1,39 @@ +# AI Usage Policy + +NeMo Guardrails welcomes responsible AI-assisted contributions. AI tools can be +useful for exploration, implementation, review, and documentation, but the human +submitter remains responsible for the contribution. + +## Contributor Responsibilities + +- Disclose AI assistance in the pull request description when AI tools create or + substantially modify code, tests, docs, issues, or comments. Include the tool + used and the extent of assistance. +- Issues must be opened manually by a human through the repository issue + templates. AI tools may help draft an issue, but agents must not submit issues + directly. +- Review, edit, and verify AI-generated content before submitting it. Do not + paste unreviewed AI output into issues, PR descriptions, code comments, docs, + or review comments. +- Understand every submitted change well enough to explain what it does, why it + is needed, and how it interacts with the surrounding code. +- Keep AI-assisted pull requests cohesive, scoped, and useful. Duplicate, + low-value, mechanical, or noisy contributions may be closed. +- Do not add AI tools as commit co-authors. Contributions should be authored by + the human submitter and must still satisfy the DCO or GPG-signing + requirements in `CONTRIBUTING.md`. + +## Safety and Privacy + +- Do not commit API keys, credentials, private endpoints, proprietary prompts, + raw provider logs, or sensitive request/response data. +- Do not use AI tools to fabricate test results, benchmark results, citations, + maintainer approvals, or compatibility claims. +- Generated media, large generated assets, or synthetic datasets require clear + provenance and maintainer alignment before inclusion. + +## Maintainer Expectations + +AI assistance does not lower the review bar. Maintainers may ask contributors to +explain, simplify, test, rewrite, or withdraw AI-assisted work that is not ready +for review. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000000..47dc3e3d86 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 294b128c60..67382e5d7d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,496 +1,222 @@ -# CONTRIBUTING GUIDELINES +# Contributing Guidelines -Welcome to the NeMo Guardrails contributing guide. We're excited to have you here and grateful for your contributions. This document provides guidelines and instructions for contributing to this project. +Welcome to NeMo Guardrails. This guide explains the contribution workflow, +local setup, validation commands, and review expectations for this repository. -> [!WARNING] -> We have recently migrated to using Poetry for dependency management and packaging. Please ensure you have Poetry installed and use it for all dependency management tasks. +Coding agents should also read [AGENTS.md](./AGENTS.md). AI-assisted public +contributions must follow [AI_POLICY.md](./AI_POLICY.md). -## Table of Contents +## Before You Contribute -- [How to Contribute](#how-to-contribute) - - [Reporting Bugs](#reporting-bugs) - - [Suggesting Enhancements and New Features](#suggesting-enhancements-and-new-features) - - [Code Contributions](#code-contributions) - - [Getting Started](#getting-started) - - [Contribution Workflow](#contribution-workflow) - - [Pull Request Checklist](#pull-request-checklist) - - [Folder Structure](#folder-structure) - - [Coding Style](#coding-style) - - [Submitting Your Work](#submitting-your-work) -- [Community and Support](#community-and-support) +- Search existing [issues](https://github.com/NVIDIA-NeMo/Guardrails/issues) + and [pull requests](https://github.com/NVIDIA-NeMo/Guardrails/pulls) before + opening anything new. +- Use the GitHub issue templates for bugs, feature requests, and documentation + issues. Blank issues are disabled. +- Use [Discussions](https://github.com/NVIDIA-NeMo/Guardrails/discussions) for + support questions and "How do I...?" questions. +- Do not open a pull request before the related issue has been triaged and + assigned to you. -# How to Contribute +Issues must be opened manually by a human using the GitHub issue templates. AI +tools may help draft or refine issue text, but agents must not open issues +directly through browser automation, the GitHub API, the `gh` CLI, or similar +tooling. The person opening the issue is responsible for reviewing, editing, and +owning the content. -You can contribute to this project in several ways, including: +## Issues and Proposals -- [Reporting Bugs](#reporting-bugs) -- [Suggesting Enhancements and New Features](#suggesting-enhancements-and-new-features) -- [Documentation Improvements](#documentation-improvements) -- [Code Contributions](#code-contributions) +Anyone is welcome to open an issue. Opening an issue does not mean you are +committing to implement it; it is fine to report a bug, propose an idea, or +share a design concern for someone else to pick up later. -## Reporting Bugs +If an issue author is not interested or available to implement the change, +another contributor may ask to take it over. Wait for maintainer assignment +before opening a PR. -### Steps to Review Before Reporting a Bug +Useful issue comments: -When preparing to report a bug, please follow these steps to ensure efficiency: - -- **Review Existing Issues**: Search the [issue tracker](https://github.com/NVIDIA-NeMo/Guardrails/issues) to confirm that the problem you’re experiencing has not been reported already. -- **Confirm the Nature of the Issue**: Ensure that what you are reporting is a genuine bug, not a support question or topic better suited for our [Discussions](https://github.com/NVIDIA-NeMo/Guardrails/discussions) page. -- **Reopen Related Issues**: If you discover a closed issue that mirrors your current experience, create a new issue and reference the closed one with a link to provide context. -- **Check Release Updates**: Look at the latest release notes to see if your issue is mentioned, along with any upgrade instructions or known issues. - -### Documenting the Problem Clearly and Thoroughly - -To ensure your issue report is easy to find and understand, follow these steps: - -- **Create a Clear, Descriptive Title**: Choose a concise and specific title that identifies the problem. -- **Detailed Reproduction Steps**: Provide a step-by-step guide to reproduce the issue. Include all necessary details to avoid ambiguity. Can you reproduce the issue following these steps? -- **Observed vs. Expected Behavior**: Describe what actually happened when you followed the reproduction steps, and explain why this behavior is problematic. Additionally, outline what you expected to happen and why this would be the correct behavior. -- **Minimal Configuration**: Share a minimal configuration that triggers the issue. If your configuration contains information that you don't like to remain on the repo, consider providing it in a [Gist](https://gist.github.com/) or an example repository after redacting any private data (e.g., private package repositories or specific names). -- **Reproducibility Details**: If the issue is intermittent, specify how often it occurs and under what conditions it typically happens. - -**Additional Context to Include**: +```text +I am opening this for tracking, but I am not planning to implement it. +``` -- **Recent Onset vs. Longstanding Issue**: Clarify whether the issue started recently (e.g., after an update) or has been persistent. If it started recently, check if you can reproduce the issue in an older version, and specify the most recent version where it did not occur. -- **Configuration and Environment Details**: +```text +I would like to work on this. +Proposed approach: <1-3 sentence summary> +Planned validation: +``` -- The version of NeMo Guardrails you are using (e.g., `nemoguardrails --version`). -- The Python version in use. -- The name and version of the operating system (e.g., Ubuntu 22.04, macOS 14.2). +```text +Is this still being worked on? If not, I would be happy to take it over. +Proposed approach: <1-3 sentence summary> +``` -> **Note**: These information are requested in the template while you are reporting the issue. +Refactors are maintainer-led and are not accepted as unsolicited PRs. If you +believe a refactor is needed, open a refactor proposal issue and wait for +maintainer feedback. Maintainers decide whether the refactor should happen, what +plan is acceptable, and who, if anyone, should be assigned to implement it. -**Ensuring Accurate Reproduction Steps**: +For work in progress, experiments, or early ideas, share the branch name and +relevant files in the issue instead of opening a premature PR. -To maximize the chances of others understanding and reproducing your issue: +## Pull Request Requirements -- Test the issue in a clean environment. +Pull requests must be linked to a triaged issue. The PR author must be assigned +to that issue before opening the PR. PRs without a linked issue, or opened by +someone who is not assigned to the linked issue, may be closed or redirected +without review. -This thorough approach helps rule out local setup issues and assists others in accurately replicating your environment for further analysis. +Before opening a PR: -## Suggesting Enhancements and New Features +- Fork the repository and create a branch from `develop`. +- Keep the PR cohesive and reviewable. +- Avoid low-value mechanical PRs such as isolated formatting churn, broad + cleanup without a clear user benefit, or typo-only sweeps. +- Use the pull request template and list the tests/checks you ran. +- Use a clear [Conventional Commit](https://www.conventionalcommits.org/) + style PR title, for example `fix: ...`, `feat: ...`, `docs: ...`, + `test: ...`, `refactor: ...`, `perf: ...`, `style: ...`, `chore: ...`, + `ci: ...`, or `revert: ...`. Use scopes when helpful, for example + `fix(server): ...`. +- Do not update `CHANGELOG.md` or `CHANGELOG-Colang.md` manually. Changelog + entries are generated by the release workflow. -This section provides instructions on how to submit enhancement or feature suggestions for NeMo Guardrails, whether they involve brand-new features or improvements to current functionality. By following these guidelines, you help maintainers and the community better understand your suggestion and identify any related discussions. +## Review Readiness -Before Submitting a Suggested Enhancement +Automated code review tools such as CodeRabbit and Greptile are part of the +pre-review workflow. A PR is ready for maintainer review only after the author +has addressed unresolved human and automated review comments. -- **Review Existing Issues**: Ensure that your suggestion has not already been submitted by checking the [issue tracker](https://github.com/NVIDIA-NeMo/Guardrails/issues) for similar ideas or proposals. +Before requesting maintainer review: -### How to Submit an Enhancement Suggestion? +- Address every automated code review comment, or reply with a clear reason why + no change is needed. +- When an automated review tool supports resolution confirmation, wait for the + tool to confirm that the issue is resolved. CodeRabbit may resolve + conversations itself; other tools, including Greptile, may require a follow-up + comment or rerun. +- Address every human reviewer comment, or reply with a clear reason why no + change is needed. +- Do not resolve human reviewer conversations unless you opened the + conversation, or the reviewer explicitly asks you to resolve it. Human review + conversations should normally be resolved by the reviewer who opened them. -Enhancement suggestions for NeMo Guardrails should be submitted through the main [issue tracker](https://github.com/NVIDIA-NeMo/Guardrails/issues), using the corresponding issue template provided. Follow these guidelines when submitting: +The repository may use a `status: ready for maintainer review` label for PRs +that have completed this author-response step. Do not request or apply that +label while human or automated review comments still need author action. -- **Create a Clear, Descriptive Title**: Choose a title that clearly identifies the nature of your enhancement. -- **Detailed Description**: Provide a comprehensive description of the proposed enhancement. Include specific steps, examples, or scenarios that illustrate how the feature would work or be implemented. -- **Current vs. Proposed Behavior**: Describe the existing behavior or functionality and explain how you would like it to change or be improved. Clarify why this new behavior or feature is beneficial to users and the project. +## AI-Assisted Contributions -By providing clear and detailed information, you make it easier for maintainers and the community to assess and discuss your proposal. +AI-assisted contributions are welcome when they meet the same quality bar as any +other contribution. If AI tools helped create or substantially modify a +contribution: -## Documentation Improvements +- Disclose the tool and extent of assistance in the PR description. +- Review and edit AI-generated text before submitting it. +- Make sure you understand every code change and can explain how it interacts + with the surrounding system. +- Do not add AI tools as commit co-authors. -Improving the project documentation is a valuable way to contribute to NeMo Guardrails. By enhancing the documentation, you help users understand the project better, learn how to use it effectively, and contribute to the project more easily. You can contribute to the documentation in several ways: +See [AI_POLICY.md](./AI_POLICY.md) for the full policy. -- **Fixing Typos and Grammar**: If you notice any typos, grammatical errors, or formatting issues in the documentation, feel free to correct them. -- **Clarifying Content**: If you find sections of the documentation that are unclear or confusing, you can propose changes to make them more understandable. -- **Adding Examples**: Providing examples and use cases can help users better understand how to use the project effectively. -- **New Content**: Creating new content such as tutorials, FAQs, Troubleshooting, etc. +## Development Setup -### Fern Documentation Workflow +NeMo Guardrails supports Python 3.10 through 3.13. Install Git, Poetry +`>=1.8,<2.0`, and the compiler/dev tools needed to build Annoy on your platform. -The migrated documentation source lives in `docs/` as MDX files and is built with Fern. Use the Makefile targets for Fern documentation work so the workflow stays consistent with the rest of the repository. +Clone the repository and install development dependencies: ```bash -make docs-fern-strict +git clone https://github.com/NVIDIA-NeMo/Guardrails.git nemoguardrails +cd nemoguardrails +poetry install --with dev ``` -Use the following targets for common Fern documentation tasks: - -| Target | Description | -| ------ | ----------- | -| `make docs-fern` | Run the Fern docs check. | -| `make docs-fern-strict` | Run the Fern docs check with the pinned Fern CLI version. | -| `make docs-fern-live` | Serve the Fern docs locally. | -| `make docs-fern-preview-watch` | Watch local changes and publish a Fern preview for the current branch. | -| `make docs-fern-generate-sdk` | Regenerate the Python SDK reference pages with Fern's library docs generator. | -| `make docs-fern-publish-staging` | Publish the Fern docs to the [staging instance](https://nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails). Only NeMo Guardrails maintainers can run this target. | -| `make docs-fern-publish-public` | Publish the Fern docs to the [public instance](https://nvidia-nemo-guardrails.docs.buildwithfern.com/nemo/guardrails), which is used for the [public documentation site](https://docs.nvidia.com/nemo/guardrails). Only NeMo Guardrails maintainers can run this target. | - -For pull requests that modify documentation, the docs build workflow checks the Fern docs and publishes a PR preview for same-repository branches. When a documentation pull request merges into `develop`, the workflow publishes the Fern docs to the [staging instance](https://nvidia-nemo-guardrails-staging.docs.buildwithfern.com/nemo/guardrails). - -Publishing the Fern docs to the public instance is a manual maintainer action after staging verification. - -The Fern CLI version is pinned in `fern/fern.config.json`. Do not run `fern upgrade` as part of normal documentation changes. - -When editing the migrated docs: - -- Edit `.mdx` files directly. -- Do not edit deleted legacy `.md` source files. -- Do not run the old conversion scripts unless you are intentionally restarting the migration from the legacy source. -- After regenerating the Python SDK reference, verify that the generated sidebar does not contain duplicate page and folder entries. - -## Code Contributions - -If you’re contributing for the first time and are searching for an issue to work on, we encourage you to check the [Contributing page](https://github.com/NVIDIA-NeMo/Guardrails/contribute) for suitable candidates. We strive to keep a selection of issues curated for first-time contributors, but sometimes there may be delays in updating. If you don’t find anything that fits, don’t hesitate to ask for guidance. -If you would like to take on an issue, feel free to comment on the issue. We are more than happy to discuss solutions on the issue. - -> **Note**: Before submitting a pull request, ensure that you have read and understood the [Contribution Workflow](#contribution-workflow) section. Always open an issue before submitting a pull request so that others can access it in future and potentially discuss the changes you plan to make. We do not accept pull requests without an associated issue. - -### Getting Started - -To get started quickly, follow the steps below. - -1. Ensure you have Python 3.10+ and [Git](https://git-scm.com/) installed on your system. You can check your Python version by running: - - ```bash - python --version - # or - python3 --version - ``` - - > Note: we suggest you use `pyenv` to manage your Python versions. You can find the installation instructions [here](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation). - -2. Clone the project repository: - - ```bash - git clone https://github.com/NVIDIA-NeMo/Guardrails.git nemoguardrails - ``` - -3. Navigate to the project directory: - - ```bash - cd nemoguardrails - ``` - -4. we use `Poetry` to manage the project dependencies. To install Poetry follow the instructions [here](https://python-poetry.org/docs/#installation): - - > Note: This project requires Poetry version >=1.8,<2.0. Please ensure you are using a compatible version before running any Poetry commands. - - Ensure you have `poetry` installed: - - ```bash - poetry --version - ``` - -5. Install the dev dependencies: - - ```bash - poetry install --with dev - ``` - - The preceding command installs pre-commit, pytest, and other development tools. - Specify `--with dev,docs` to add the dependencies for building the documentation. - -6. If needed, you can install extra dependencies as below: - - ```bash - poetry install --extras "openai tracing" - # or Alternatively using the following command - poetry install -E openai -E tracing - - ``` - - to install all the extras: - - ```bash - poetry install --all-extras - ``` - - > **Note**: `dev` is not part of the extras but it is an optional dependency group, so you need to install it as instructed above. - -7. Set up pre-commit hooks: - - ``` - pre-commit install - ``` - - This will ensure that the pre-commit checks, including Black, are run before each commit. - -8. Run the tests: - - ```bash - poetry run pytest - ``` - - This will run the test suite to ensure everything is set up correctly. - -> **Note**: You should use `poetry run` to run commands within the virtual environment. If you want to avoid prefixing commands with `poetry run`, you can activate the environment using `poetry shell`. This will start a new shell with the virtual environment activated, allowing you to run commands directly. - -### Contribution Workflow - -This project follows the [GitFlow](https://nvie.com/posts/a-successful-git-branching-model/) branching model which involves the use of several branch types: - -- `main`: Latest stable release branch. -- `develop`: Development branch for integrating features. -- `feature/...`: Feature branches for new features and non-emergency bug fixes. -- `release/...`: Release branches for the final versions published to PyPI. -- `hotfix/...`: Hotfix branches for emergency bug fixes. - -Additionally, we recommend the use of `docs/...` documentation branches for contributions that update only the project documentation. You can find a comprehensive guide on using GitFlow here: [GitFlow Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow). - -To contribute your work, follow the following process: - -1. **Fork the Repository**: Fork the project repository to your GitHub account. -2. **Clone Your Fork**: Clone your fork to your local machine. -3. **Create a Feature Branch**: Create a branch from the `develop` branch. -4. **Develop**: Make your changes locally and commit them. -5. **Push Changes**: Push your changes to your GitHub fork. -6. **Open a Pull Request (PR)**: Create a PR against the main project's `develop` branch. - -### Pull Request Checklist - -Before submitting your Pull Request (PR) on GitHub, please ensure you have completed the following steps. This checklist helps maintain the quality and consistency of the codebase. - -1. **Documentation**: - - Ensure that all new code is properly documented. Update the README, API documentation, and any other relevant documentation if your changes introduce new features or change existing functionality. - -2. **Tests Passing**: - - Run the project's test suite to make sure all tests pass. Include new tests if you are adding new features or fixing bugs. If applicable, ensure your code is compatible with different Python versions or environments. - - You can run the tests using `pytest`: - - ```bash - poetry run pytest - ``` - - Or using `make`: - - ```bash - make tests - ``` - - You can use `tox` to run the tests for the supported Python versions: - - ```bash - tox - ``` - - We recommend you to run the test coverage to ensure that your changes are well tested: - - ```bash - make test_coverage - ``` - -3. **Changelog Updated**: - - Update the `CHANGELOG.md` file with a brief description of your changes, following the existing format. This is important for keeping track of new features, improvements, and bug fixes. - - > **Note**: If your new feature concerns Colang, please update the `CHANGELOG_Colang.md` file. - -4. **Code Style and Quality**: +Install documentation dependencies when working on docs: - Adhere to the project's coding style guidelines. Keep your code clean and readable. - -5. **Commit Guidelines**: - - Follow the commit message guidelines, ensuring clear and descriptive commit messages. Sign your commits as per the Developer Certificate of Origin (DCO) or GPG-sign them for verification. - -6. **No Merge Conflicts**: - - Before submitting, rebase your branch onto the latest version of the `develop` branch to ensure your PR can be merged smoothly. - -7. **Self Review**: - - Self-review your changes and compare them to the contribution guidelines to ensure you haven't missed anything. - -By following this checklist, you help streamline the review process and increase the chances of your contribution being merged without significant revisions. Your MR/PR will be reviewed by at least one of the maintainers, who may request changes or further details. - -### Folder Structure - -The project is structured as follows: - -``` -. -├── chat-ui -├── docs -├── examples -├── nemoguardrails -├── qa -├── tests +```bash +poetry install --with dev,docs ``` -- `chat-ui`: includes a static build of the Guardrails Chat UI. This UI is forked from [https://github.com/mckaywrigley/chatbot-ui](https://github.com/mckaywrigley/chatbot-ui) and is served by the NeMo Guardrails server. The source code for the Chat UI is not included as part of this repository. -- `docs`: includes the official documentation of the project. -- `examples`: various examples, including guardrails configurations (example bots, using different LLMs and others), notebooks, or Python scripts. -- `nemoguardrails`: the source code for the main `nemoguardrails` package. -- `qa`: a set of scripts the QA team uses. -- `tests`: the automated tests set that runs automatically as part of the CI pipeline. - -### Coding Style - -We follow the [Black](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) coding style for this project. To maintain consistent code quality and style, the [pre-commit](https://pre-commit.com) framework is used. This tool automates the process of running various checks, such as linters and formatters, before each commit. It helps catch issues early and ensures all contributions adhere to our coding standards. - -### Setting Up Pre-Commit - -1. **Install Pre-Commit**: - - First, you need to install pre-commit on your local machine. It can be installed via `poetry`: - - ```bash - poetry add pre-commit - ``` - - Alternatively, you can use other installation methods as listed in the [pre-commit installation guide](https://pre-commit.com/#install). - -2. **Configure Pre-Commit in Your Local Repository**: - - In the root of the project repository, there should be a [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) file which contains the configuration and the hooks we use. Run the following command in the root of the repository to set up the git hook scripts: - - ```bash - pre-commit install - ``` - -3. **Running Pre-Commit** - - **Automatic Checks**: Once `pre-commit` is installed, the configured hooks will automatically run on each Git commit. If any changes are necessary, the commit will fail, and you'll need to make the suggested changes. - - **Manual Run**: You can manually run all hooks against all the files with the following command: - - ```bash - pre-commit run --all-files - ``` - - To do steps 2 and 3 in one command: - - ```bash - make pre_commit - ``` - -### Installing Dependencies Without Modifying `pyproject.toml` - -To install a dependency using Poetry without adding it to the `pyproject.toml` file, you can use `pip` within the Poetry-managed virtual environment. Here's how to do it: - -1. **Activate the Poetry virtual environment**: - - Run `poetry shell` to activate the virtual environment managed by Poetry. - - > **Note**: If you don't want to activate the virtual environment, you can use `poetry run` to run commands within the virtual environment. - -2. **Install the package using `pip`**: - - Once inside the virtual environment, you can use `pip` to install the package without affecting the `pyproject.toml`. For example: - - ```bash - pip install - # or if the virtual environment is not activated - poetry run pip install - ``` - -This will install the package only in the virtual environment without tracking it in `pyproject.toml`. +Valid optional extras are `sdd`, `eval`, `gcp`, `tracing`, `jailbreak`, +`multilingual`, `server`, `chat-ui`, and `all`. For example: -This method is useful when you need a package temporarily or for personal development tools that you don't want to be part of your project's formal dependencies. - -**Important Considerations**: - -- Using `pip` directly inside a Poetry-managed environment bypasses Poetry's dependency resolution, so be cautious of potential conflicts with other dependencies. -- This approach does not update the lock file (`poetry.lock`), meaning these changes are not reproducible for others or on different environments unless manually replicated. -- If you decided to add the dependency permanently, you should add it to the `pyproject.toml` file using Poetry's `add` command. - -This workaround is commonly used because Poetry currently does not have a built-in feature to install packages without modifying `pyproject.toml`. - -## Jupyter Notebook Documentation - -For certain features, you can provide documentation in the form of a Jupyter notebook. In addition to the notebook, we also require that you generate a README.md file next to the Jupyter notebook, with the same content. To achieve this, follow the following process: - -1. Place the jupyter notebook in a separate sub-folder. - -2. Install `nbdoc`: - - ```bash - poetry run pip install nbdoc - ``` - -3. Use the `build_notebook_docs.py` script from the root of the project to perform the conversion: - - ```bash - poetry run python build_notebook_docs.py PATH/TO/SUBFOLDER - ``` - -### Submitting Your Work - -We require that all contributions are certified under the terms of the Developer Certificate of Origin (DCO), Version 1.1. This certifies that the contribution is your original work or you have the right to submit it under the same or compatible license. Any public contribution that contains commits that are not signed off will not be accepted. - -To simplify the process, we accept GPG-signed commits as fulfilling the requirements of the DCO. - -#### Why GPG Signatures? - -A GPG-signed commit provides cryptographic assurance that the commit was made by the holder of the corresponding private key. By configuring your commits to be signed by GPG, you not only enhance the security of the repository but also implicitly certify that you have the rights to submit the work under the project's license and agree to the DCO terms. - -#### Setting Up Git for Signed Commits - -1. **Generate a GPG key pair**: - - If you don't already have a GPG key, you can generate a new GPG key pair by following the instructions here: [Generating a new GPG key](https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key). - -2. **Add your GPG key to your GitHub/GitLab account**: - - After generating your GPG key, add it to your GitHub account by following these steps: [Adding a new GPG key to your GitHub account](https://docs.github.com/en/authentication/managing-commit-signature-verification/adding-a-gpg-key-to-your-github-account). - -3. **Configure Git to sign commits:** - - Tell Git to use your GPG key by default for signing your commits: - - ```bash - git config --global user.signingkey YOUR_GPG_KEY_ID - ``` - -4. **Sign commits**: - - Sign individual commits using the `-S` flag +```bash +poetry install --with dev -E server -E tracing +``` - ```bash - git commit -S -m "Your commit message" - ``` +For temporary local investigation tools, use the Poetry-managed environment +without modifying project dependencies: - Or, enable commit signing by default (recommended): +```bash +poetry run pip install +``` - ```bash - git config --global commit.gpgsign true - ``` +Do not commit environment-only dependency changes. -**Troubleshooting and Help**: If you encounter any issues or need help with setting up commit signing, please refer to the [GitHub documentation on signing commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). -Feel free to contact the project maintainers if you need further assistance. +## Validation -#### Developer Certificate of Origin (DCO) +Run Python commands through Poetry. -To ensure the quality and legality of the code base, all contributors are required to certify the origin of their contributions under the terms of the Developer Certificate of Origin (DCO), Version 1.1: +| Task | Command | +| --- | --- | +| Focused tests | `make test TEST=path/to/test_file.py::test_name` | +| Full test suite | `make test` | +| Supported Python versions | `poetry run tox` | +| Pre-commit hooks | `poetry run pre-commit run --all-files` | +| Docs check | `make docs-fern` | +| Package coverage | `make test-coverage` | - ``` - Developer Certificate of Origin - Version 1.1 +Run the smallest meaningful test set first, then broaden validation when the +change touches shared runtime behavior, public APIs, packaging, server behavior, +tracing, or docs. - Copyright (C) 2004, 2006 The Linux Foundation and its contributors. - 1 Letterman Drive - Suite D4700 - San Francisco, CA, 94129 +Set up local pre-commit hooks if you want checks to run before every commit: - Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. +```bash +poetry run pre-commit install +``` - Developer's Certificate of Origin 1.1 +The pre-commit configuration runs Ruff, Ruff format, license-header insertion, +and Pyright. - By making a contribution to this project, I certify that: +## Documentation and Notebooks - (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or +Update documentation when changing user-visible behavior, public APIs, +configuration syntax, examples, or installation requirements. - (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or +Documentation lives in `docs/` as MDX and is built with Fern. Edit the `.mdx` +files directly and check changes with `make docs-fern` (`make docs-fern-live` +serves locally; `make docs-check-links` validates links). The Fern CLI version +is pinned in `fern/fern.config.json`; do not run `fern upgrade` as part of normal +documentation changes. - (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. +For notebook documentation, place the notebook in its own folder and generate a +matching `README.md` with: - (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. - ``` +```bash +poetry run python build_notebook_docs.py PATH/TO/SUBFOLDER +``` -#### Why the DCO is Important +Important: `build_notebook_docs.py` currently runs broad git staging and +pre-commit commands. Use a clean worktree before running it. Coding agents +should not run it unless explicitly asked. -The DCO helps to ensure that contributors have the right to submit their contributions under the project's license, protecting both the contributors and the project. It's a lightweight way to manage contributions legally without requiring a more cumbersome Contributor License Agreement (CLA). +## Commit Signing -#### Summary +Public contributions must satisfy the Developer Certificate of Origin (DCO). +Use one of these options: -- A GPG-signed commit will be accepted as a declaration that you agree to the terms of the DCO. -- Alternatively, you can manually add a "Signed-off-by" line to your commit messages to comply with the DCO. +- Submit GPG-signed commits. +- Add a `Signed-off-by` line to commit messages. -By following these guidelines, you help maintain the integrity and legal compliance of the project. +PR titles and commit messages should follow the project commit convention +described above. A GPG-signed commit is accepted as a declaration that you agree +to the DCO terms. For details, see the +[Developer Certificate of Origin](https://developercertificate.org/) and +[GitHub's signing commits documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). ## Community and Support -For general questions or discussion about the project, use the [discussions](https://github.com/NVIDIA-NeMo/Guardrails/discussions) section. - -Thank you for contributing to NeMo Guardrails! +For general questions and discussion, use +[GitHub Discussions](https://github.com/NVIDIA-NeMo/Guardrails/discussions). diff --git a/nemoguardrails/AGENTS.md b/nemoguardrails/AGENTS.md new file mode 100644 index 0000000000..58464752b5 --- /dev/null +++ b/nemoguardrails/AGENTS.md @@ -0,0 +1,122 @@ +# AGENTS.md + +Subtree guidance for the `nemoguardrails/` package. This supplements the +repository-root `AGENTS.md`, `CONTRIBUTING.md`, and `AI_POLICY.md`; all +root-level rules still apply. The rules below are the runtime, public-API, and +provider-integration specifics that matter when editing this package. + +## Architecture Map + +- `Guardrails` (`guardrails/guardrails.py`) is the modern entry point; it + delegates to `IORails` (`guardrails/iorails.py`, input/output rails dispatched + through the engine registry) or to the legacy `LLMRails` + (`rails/llm/llmrails.py`, the full event-driven Colang pipeline). +- `RailsConfig` (`rails/llm/config.py`) is the user-facing config, loaded via + `from_path`/`from_content`. +- Colang has two runtimes, 1.0 (`colang/v1_0/`) and 2.x (`colang/v2_x/`), + dispatched by `colang/__init__.py`; actions resolve through + `actions/action_dispatcher.py`. +- LLM access goes through the framework/provider abstraction (`llm/frameworks/`, + `types.py`): the default OpenAI-compatible client or the LangChain framework. +- Built-in rails live in `library/` (optional, lazily imported); the FastAPI + server is in `server/`; request-scoped state lives in `context.py`. + +## Code Changes + +- Preserve public APIs unless the task explicitly changes them. Treat public + imports, constructor signatures, documented methods, config schemas, server + request/response shapes, Colang behavior, examples, and shipped defaults as + compatibility-sensitive. +- Keep sync and async API behavior aligned for `LLMRails`, `Guardrails`, and + related public methods. +- Keep optional providers, frameworks, extras, and secret-bearing integrations + optional. Do not move integration dependencies into the default install path + unless the task is specifically about packaging policy. +- When changing API request or response shapes, keep required fields explicit + and never mirror API keys, credentials, or provider secrets back in response + bodies. Secrets belong in headers, environment variables, or local + configuration paths. +- Route LLM calls through existing framework/model abstractions and helpers such + as `nemoguardrails.actions.llm.utils.llm_call` unless the surrounding code + already establishes a more specific path. Avoid ad hoc provider calls that + bypass shared parameter handling, tracing, metrics, or streaming behavior. +- For OpenAI-compatible providers, prefer the built-in default framework and + OpenAI-compatible client path. Use the LangChain framework only for engines + that need LangChain or when the task explicitly changes LangChain behavior. +- Treat HTTP header names as case-insensitive. Only normalize or compare header + values case-insensitively when the relevant HTTP spec or provider contract + defines them that way. +- Avoid broad filesystem walks, import-time side effects, and global state + changes in runtime paths unless the surrounding code already establishes that + pattern. +- Wrap provider/LLM failures in the domain exceptions in + `nemoguardrails/exceptions.py` (`LLMCallException`, `LLMClientError` + subclasses) and re-raise with `from`; do not raise bare exceptions. +- Use a module-level `log = logging.getLogger(__name__)`; never `print`. +- Sync public methods delegate to their `_async` twin via + `get_or_create_event_loop()` and must raise if called inside a running loop; + keep the logic in the async method. +- The public surface is what top-level `__all__` exports. Domain types in + `types.py` are plain dataclasses (keep them dependency-free); config models use + Pydantic with `@model_validator` and `ConfigDict(extra="forbid")`. +- Deprecate with `warnings.warn(..., DeprecationWarning, stacklevel=...)` and + keep the old path working. + +## NeMo Guardrails Invariants + +- Preserve non-text model metadata such as reasoning content, usage data, finish + reasons, request IDs, and streamed metadata chunks. Do not drop reasoning-only + or usage-only chunks just because `delta_content` or message content is empty. +- Keep observability signals independently configurable. Tracing, metrics, logs, + and anonymous usage telemetry have different contracts and should not be + enabled, disabled, or configured as a single implicit bundle. +- Mark experimental behavior clearly in docs and keep it isolated from stable + contracts. + +## Testing + +- Test config-driven behavior against a real `RailsConfig` (for example + `RailsConfig.from_content` with YAML), not `SimpleNamespace` or attribute + stubs, when adding or wiring a config field. Stubbing the whole config tree + does not validate the wiring you are adding. +- For metadata or stats propagation changes, assert on the actual propagation + targets (for example both `LLMCallInfo` fields and `LLMStats` counters) and + reset every context variable the code path touches in fixtures. +- Mock LLMs with the project's test doubles, not bespoke mocks: `FakeLLMModel` + (`nemoguardrails/testing/fake_model.py`) for deterministic responses and token + usage, and the `TestChat` harness (`nemoguardrails/testing/chat_harness.py`, + `>>`/`<<`) for end-to-end rail tests. +- Mock provider HTTP with `pytest-httpx` (`httpx_mock`) and set secrets via + `monkeypatch`. There is no global live-test mode; gate any real-network test + behind an explicit skip. + +## Adding A Provider Or Library Integration + +When adding a new optional third-party integration (LLM provider, embedding +provider, or similar library): + +- Keep the dependency optional: import the third-party package lazily inside + `__init__` or the method that needs it, wrapped in a `try/except ImportError` + that names the poetry extra to install. Never import it at module top level in + a way that breaks core import when the package is absent. +- Add the dependency as an optional dependency under a poetry extra in + `pyproject.toml`; do not add it to the default runtime dependencies. +- Packaging-patch hygiene: read the current `pyproject.toml` and `poetry.lock` + first, anchor edits to lines that actually exist, and regenerate the lock with + project tooling. Do not generate dependency edits from memory or templates; if + the lock cannot be regenerated, stop and flag that it will be inconsistent. +- Follow the existing provider pattern. For an embedding provider, mimic + `nemoguardrails/embeddings/providers/openai.py`: subclass the `EmbeddingModel` + ABC, set `engine_name`, implement `encode()` and `encode_async()`, and register + the class with one `register_embedding_provider(...)` call in + `providers/__init__.py`. For an LLM provider, implement the `LLMModel` protocol + and route through the default OpenAI-compatible framework unless LangChain is + required. +- Tests must not call the live service: mock the client (`MagicMock`/`AsyncMock` + or httpx fixtures, as in `tests/test_embeddings_openai.py`); guard any + real-network test behind an explicit skip (there is no global live-test mode). +- Document the engine/provider name, the required extra, and the expected API + keys or environment variables, and note whether it uses the default framework + or LangChain. +- Keep the registration name and constructor signature consistent with sibling + providers. diff --git a/nemoguardrails/CLAUDE.md b/nemoguardrails/CLAUDE.md new file mode 120000 index 0000000000..47dc3e3d86 --- /dev/null +++ b/nemoguardrails/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 30674bc9e1..84be813324 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,5 +18,4 @@ markers = testpaths = tests - docs/colang-2/examples benchmark/tests From 73d9627e4c5e08ec8b4916912f3701a75dc28d3b Mon Sep 17 00:00:00 2001 From: Tim Gasser <200644301+tgasser-nv@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:17:46 -0500 Subject: [PATCH 05/21] feat(iorails): Tool-calling - non-streaming tool calling to main LLM (#2016) --- nemoguardrails/guardrails/guardrails_types.py | 5 +- nemoguardrails/guardrails/iorails.py | 64 +++++-- nemoguardrails/guardrails/model_engine.py | 27 ++- tests/guardrails/test_iorails.py | 98 ++++++++++- tests/guardrails/test_iorails_streaming.py | 26 +++ tests/guardrails/test_model_engine.py | 160 +++++++++++++++--- 6 files changed, 330 insertions(+), 50 deletions(-) diff --git a/nemoguardrails/guardrails/guardrails_types.py b/nemoguardrails/guardrails/guardrails_types.py index 73a79e7a25..1aea77159e 100644 --- a/nemoguardrails/guardrails/guardrails_types.py +++ b/nemoguardrails/guardrails/guardrails_types.py @@ -18,9 +18,10 @@ from contextvars import ContextVar, Token from dataclasses import dataclass from enum import Enum -from typing import TypeAlias +from typing import Any, TypeAlias -LLMMessage: TypeAlias = dict[str, str] # e.g. {"role": "user", "content": "What can you do?"} +# LLMMessage can contain role/content, plus optional tool_calls / tool_call_id / name; content may be None +LLMMessage: TypeAlias = dict[str, Any] LLMMessages: TypeAlias = list[LLMMessage] diff --git a/nemoguardrails/guardrails/iorails.py b/nemoguardrails/guardrails/iorails.py index 7633141dc5..99fc3e6370 100644 --- a/nemoguardrails/guardrails/iorails.py +++ b/nemoguardrails/guardrails/iorails.py @@ -65,7 +65,7 @@ from nemoguardrails.rails.llm.options import GenerationOptions from nemoguardrails.streaming import END_OF_STREAM, StreamingHandler from nemoguardrails.tracing.constants import GuardrailsAttributes -from nemoguardrails.types import LLMModel, LLMResponse +from nemoguardrails.types import LLMModel, LLMResponse, ToolCall if TYPE_CHECKING: from opentelemetry.trace import Span @@ -89,6 +89,42 @@ _GENERATION_ERROR_TYPE = "generation_error" +def _serialize_tool_calls(tool_calls: list[ToolCall]) -> list[dict]: + """Serialize ToolCall objects to OpenAI /chat/completions shape. + + ``function.arguments`` is emitted as a JSON string (OpenAI-native) rather + than the canonical dict carried internally, so the output round-trips + through OpenAI-compatible clients. + """ + return [ + { + "id": tool_call.id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": json.dumps(tool_call.function.arguments), + }, + } + for tool_call in tool_calls + ] + + +def _build_assistant_message(content: str, tool_calls: Optional[list[ToolCall]]) -> LLMMessage: + """Build the assistant message returned by ``generate``. + + Without tool calls this is the existing ``{"role", "content"}`` shape. With + tool calls present, the calls are serialized to OpenAI shape and ``content`` + is set to ``None`` when empty, matching the OpenAI assistant-message contract. + """ + if not tool_calls: + return {"role": "assistant", "content": content} + return { + "role": "assistant", + "content": content or None, + "tool_calls": _serialize_tool_calls(tool_calls), + } + + class IORails(BaseGuardrails): """Workflow engine for accelerated Input/Output rails inference.""" @@ -348,6 +384,7 @@ async def _do_generate( if options and isinstance(options, dict): options = GenerationOptions(**options) if isinstance(options, GenerationOptions) and options.llm_params: + # Pass llm_params (including tool definitions) unchanged llm_kwargs = options.llm_params if self._speculative_generation: @@ -367,21 +404,27 @@ async def _do_generate( reasoning_content = response.reasoning or _extract_and_remove_think_tags(response) response_text = response.content - # Check output rails - log.info("[%s] Running output rails", req_id) - output_result = await self.rails_manager.is_output_safe(messages, response_text) - if not output_result.is_safe: - log.info("[%s] Output blocked: %s", req_id, output_result.reason) - if self._metrics_enabled: - record_request_blocked(RailDirection.OUTPUT) - return {"role": "assistant", "content": REFUSAL_MESSAGE} + # Output rails check the final answer, not reasoning traces. + # Reasoning is re-attached as tags only below so reasoning intentionally bypasses output + # rails, matching LLMRails. + # A tool-call-only response skips output rails (no text to check) + # Tool calls have their own `ToolOutputRails` set of rails separate to `OutputRails` + is_tool_call_only = bool(response.tool_calls) and not response_text + if not is_tool_call_only: + log.info("[%s] Running output rails", req_id) + output_result = await self.rails_manager.is_output_safe(messages, response_text) + if not output_result.is_safe: + log.info("[%s] Output blocked: %s", req_id, output_result.reason) + if self._metrics_enabled: + record_request_blocked(RailDirection.OUTPUT) + return {"role": "assistant", "content": REFUSAL_MESSAGE} # TODO: Support returning GenerationResponse `reasoning_content` to match LLMRails # For now, embed the reasoning on the content with think-tags if reasoning_content: response_text = f"{reasoning_content}\n" + response_text - return {"role": "assistant", "content": response_text} + return _build_assistant_message(response_text, response.tool_calls) async def _do_generate_sequential( self, messages: LLMMessages, req_id: str, llm_kwargs: dict @@ -555,6 +598,7 @@ def stream_async( if options and isinstance(options, dict): options = GenerationOptions(**options) if isinstance(options, GenerationOptions) and options.llm_params: + # Pass llm_params (including tool definitions) unchanged llm_kwargs = options.llm_params streaming_handler = StreamingHandler(include_metadata=include_metadata) diff --git a/nemoguardrails/guardrails/model_engine.py b/nemoguardrails/guardrails/model_engine.py index f962876f4d..307ef7b4da 100644 --- a/nemoguardrails/guardrails/model_engine.py +++ b/nemoguardrails/guardrails/model_engine.py @@ -39,7 +39,7 @@ from nemoguardrails.guardrails.base_engine import BaseEngine from nemoguardrails.guardrails.guardrails_types import LLMMessages, get_request_id, truncate from nemoguardrails.rails.llm.config import Model -from nemoguardrails.types import LLMResponse, LLMResponseChunk, UsageInfo +from nemoguardrails.types import ChatMessage, LLMResponse, LLMResponseChunk, UsageInfo log = logging.getLogger(__name__) @@ -82,8 +82,12 @@ def _parse_chat_completion(response: dict) -> LLMResponse: """Convert a /v1/chat/completions response dict into an LLMResponse. Reasoning is read from ``message.reasoning_content`` when the provider - exposes it (NIM, DeepSeek-style). Tool calls are out of scope for this - PR series and are not currently surfaced. + exposes it (NIM, DeepSeek-style). Tool calls are parsed from + ``message.tool_calls`` (OpenAI shape) into ``LLMResponse.tool_calls`` via + ``ChatMessage.from_dict``, which normalizes JSON-string arguments into a + dict. ``content`` is ``None`` on a tool-call-only response and is + normalized to an empty string; a ``None`` content with no tool calls is + treated as a malformed response. """ try: choice = response["choices"][0] @@ -92,13 +96,19 @@ def _parse_chat_completion(response: dict) -> LLMResponse: except (KeyError, IndexError, TypeError) as exc: raise ValueError(f"Unexpected /v1/chat/completions response shape: {exc}") from exc - if not isinstance(content, str): - if content is None and message.get("tool_calls"): - raise ValueError( - "Tool-call-only responses are not yet supported by IORails (message contains tool_calls but no content)" - ) + raw_tool_calls = message.get("tool_calls") + + if content is None: + # A tool-call-only response legitimately carries content=None; a null + # content with no tool calls is malformed. + if not raw_tool_calls: + raise ValueError("Expected string content, got NoneType") + content = "" + elif not isinstance(content, str): raise ValueError(f"Expected string content, got {type(content).__name__}") + tool_calls = ChatMessage.from_dict(message).tool_calls if raw_tool_calls else None + reasoning = message.get("reasoning_content") or None usage = _parse_usage(response["usage"]) if response.get("usage") else None @@ -106,6 +116,7 @@ def _parse_chat_completion(response: dict) -> LLMResponse: return LLMResponse( content=content, reasoning=reasoning, + tool_calls=tool_calls, model=response.get("model"), finish_reason=choice.get("finish_reason"), request_id=response.get("id"), diff --git a/tests/guardrails/test_iorails.py b/tests/guardrails/test_iorails.py index 2623d3ffa4..0a4410365a 100644 --- a/tests/guardrails/test_iorails.py +++ b/tests/guardrails/test_iorails.py @@ -22,12 +22,12 @@ import pytest_asyncio from nemoguardrails import Guardrails -from nemoguardrails.guardrails.guardrails_types import RailResult +from nemoguardrails.guardrails.guardrails_types import RailDirection, RailResult from nemoguardrails.guardrails.iorails import REFUSAL_MESSAGE, IORails from nemoguardrails.guardrails.model_engine import ModelEngine from nemoguardrails.rails.llm.config import RailsConfig from nemoguardrails.rails.llm.options import GenerationOptions -from nemoguardrails.types import LLMResponse, LLMResponseChunk +from nemoguardrails.types import LLMResponse, LLMResponseChunk, ToolCall, ToolCallFunction from tests.guardrails.test_data import CONTENT_SAFETY_CONFIG, NEMOGUARDS_CONFIG @@ -164,6 +164,22 @@ async def test_unsafe_output(self, iorails): iorails.engine_registry.model_call.assert_called_once_with("main", messages) iorails.rails_manager.is_output_safe.assert_called_once_with(messages, llm_response) + @pytest.mark.asyncio + async def test_unsafe_output_records_block_metric_when_metrics_enabled(self, iorails): + """When metrics are enabled, an output-rails block records an OUTPUT block metric.""" + iorails._metrics_enabled = True + messages = [{"role": "user", "content": "hi"}] + + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="unsafe")) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=False, reason="blocked")) + + with patch("nemoguardrails.guardrails.iorails.record_request_blocked") as record_blocked: + result = await iorails.generate_async(messages) + + assert result == {"role": "assistant", "content": REFUSAL_MESSAGE} + record_blocked.assert_called_once_with(RailDirection.OUTPUT) + @pytest.mark.asyncio async def test_dict_options_forwarded(self, iorails): """Dict options are converted to GenerationOptions and forwarded.""" @@ -188,6 +204,84 @@ async def test_generate_async_propagates_exception(self, iorails): await iorails.generate_async([{"role": "user", "content": "hi"}]) +class TestToolCalling: + """Test tool-call forwarding (request body) and return (assistant message).""" + + @pytest.mark.asyncio + async def test_tools_in_llm_params_forwarded_to_model(self, iorails): + """Tool definitions in options.llm_params are forwarded to model_call unchanged.""" + messages = [{"role": "user", "content": "weather?"}] + tool = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} + options = GenerationOptions(llm_params={"tools": [tool], "tool_choice": "auto", "parallel_tool_calls": True}) + + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="ok")) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + + await iorails.generate_async(messages, options=options) + + iorails.engine_registry.model_call.assert_called_once_with( + "main", messages, tools=[tool], tool_choice="auto", parallel_tool_calls=True + ) + + @pytest.mark.asyncio + async def test_tool_calls_returned_on_assistant_message(self, iorails): + """Tool calls from the model are serialized OpenAI-native (JSON-string arguments).""" + messages = [{"role": "user", "content": "weather?"}] + tool_calls = [ + ToolCall( + id="call_1", + type="function", + function=ToolCallFunction(name="get_weather", arguments={"city": "Paris"}), + ) + ] + + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="", tool_calls=tool_calls)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + + result = await iorails.generate_async(messages) + + assert result["role"] == "assistant" + assert result["content"] is None + assert result["tool_calls"] == [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + ] + + @pytest.mark.asyncio + async def test_output_rails_skipped_for_tool_call_only_response(self, iorails): + """A tool-call-only response (no text) skips the content output rails.""" + messages = [{"role": "user", "content": "weather?"}] + tool_calls = [ToolCall(id="c1", type="function", function=ToolCallFunction(name="f", arguments={}))] + + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock(return_value=LLMResponse(content="", tool_calls=tool_calls)) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + + result = await iorails.generate_async(messages) + + iorails.rails_manager.is_output_safe.assert_not_called() + assert result["tool_calls"][0]["function"]["name"] == "f" + + @pytest.mark.asyncio + async def test_text_with_tool_calls_runs_output_rails_and_returns_both(self, iorails): + """When a response has text and tool calls, output rails run on the text and both are returned.""" + messages = [{"role": "user", "content": "hi"}] + tool_calls = [ToolCall(id="c1", type="function", function=ToolCallFunction(name="f", arguments={}))] + + iorails.rails_manager.is_input_safe = AsyncMock(return_value=RailResult(is_safe=True)) + iorails.engine_registry.model_call = AsyncMock( + return_value=LLMResponse(content="some text", tool_calls=tool_calls) + ) + iorails.rails_manager.is_output_safe = AsyncMock(return_value=RailResult(is_safe=True)) + + result = await iorails.generate_async(messages) + + iorails.rails_manager.is_output_safe.assert_called_once_with(messages, "some text") + assert result["content"] == "some text" + assert result["tool_calls"][0]["function"]["name"] == "f" + + class TestIORailsLifecycle: """Test IORails start/stop lifecycle management.""" diff --git a/tests/guardrails/test_iorails_streaming.py b/tests/guardrails/test_iorails_streaming.py index 8dd3665aa6..d473dc7ae4 100644 --- a/tests/guardrails/test_iorails_streaming.py +++ b/tests/guardrails/test_iorails_streaming.py @@ -209,6 +209,32 @@ async def test_speculative_generation_streaming_warning_recorded_once(self): matching_warnings = [warning for warning in caught if str(warning.message) == _SPECULATIVE_STREAM_WARNING] assert len(matching_warnings) == 1 + @pytest.mark.asyncio + async def test_tools_in_llm_params_forwarded_on_stream_async(self, iorails_input_only): + """Tool definitions in llm_params are forwarded to the streaming LLM call unchanged. + + Streaming tool-call delta parsing is deferred to a later PR; the request-side + forwarding works today because llm_params passes through untouched. + """ + captured_kwargs = {} + + async def capturing_stream(model_type, messages, **kwargs): + """Mock stream that records kwargs.""" + captured_kwargs.update(kwargs) + yield LLMResponseChunk(delta_content="ok") + + _wire_mocks(iorails_input_only, stream=capturing_stream) + tool = {"type": "function", "function": {"name": "get_weather"}} + options = GenerationOptions(llm_params={"tools": [tool], "tool_choice": "auto"}) + + chunks = await _collect( + iorails_input_only.stream_async(messages=[{"role": "user", "content": "hi"}], options=options) + ) + + assert "".join(chunks) == "ok" + assert captured_kwargs.get("tools") == [tool] + assert captured_kwargs.get("tool_choice") == "auto" + class TestStreamAsyncNoOutputRails: """Test streaming when there are no output rails -- chunks flow straight through.""" diff --git a/tests/guardrails/test_model_engine.py b/tests/guardrails/test_model_engine.py index 261a8e0132..02e4b40874 100644 --- a/tests/guardrails/test_model_engine.py +++ b/tests/guardrails/test_model_engine.py @@ -1137,7 +1137,7 @@ async def test_raises_on_missing_content(self): @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) @pytest.mark.asyncio async def test_raises_on_null_content(self): - """chat_completion() raises ModelEngineError for unsupported tool-calls""" + """content=None with no tool_calls is malformed; chat_completion() raises ModelEngineError.""" engine = ModelEngine(_make_model()) engine.call = AsyncMock(return_value={"choices": [{"message": {"content": None}}]}) @@ -1146,8 +1146,8 @@ async def test_raises_on_null_content(self): @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) @pytest.mark.asyncio - async def test_raises_clearer_error_for_tool_call_only_response(self): - """Tool-call-only responses (content=None, tool_calls set) get a scope-specific error.""" + async def test_parses_tool_call_only_response(self): + """Tool-call-only responses (content=None, tool_calls set) are parsed, not rejected.""" engine = ModelEngine(_make_model()) engine.call = AsyncMock( return_value={ @@ -1160,17 +1160,25 @@ async def test_raises_clearer_error_for_tool_call_only_response(self): { "id": "call_abc", "type": "function", - "function": {"name": "calculate", "arguments": '{"expr":"2+2"}'}, + "function": {"name": "calculate", "arguments": '{"expr": "2+2"}'}, } ], - } + }, + "finish_reason": "tool_calls", } ] } ) - with pytest.raises(ModelEngineError, match="Tool-call-only responses are not yet supported"): - await engine.chat_completion([{"role": "user", "content": "Hi"}]) + result = await engine.chat_completion([{"role": "user", "content": "Hi"}]) + + assert result.content == "" + assert result.finish_reason == "tool_calls" + assert result.tool_calls is not None + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].id == "call_abc" + assert result.tool_calls[0].function.name == "calculate" + assert result.tool_calls[0].function.arguments == {"expr": "2+2"} class TestParseChatCompletion: @@ -1206,32 +1214,128 @@ def test_raises_on_malformed_response(self): with pytest.raises(ValueError, match="Unexpected /v1/chat/completions response shape"): _parse_chat_completion({}) - def test_raises_on_non_string_content(self): - """Non-string content raises ValueError.""" - with pytest.raises(ValueError, match="Expected string content"): + def test_raises_on_null_content_without_tool_calls(self): + """content=None with no tool_calls is malformed and raises ValueError.""" + with pytest.raises(ValueError, match="Expected string content, got NoneType"): _parse_chat_completion({"choices": [{"message": {"content": None}}]}) - def test_raises_specific_error_for_tool_call_only_response(self): - """When content is None and tool_calls is set, raise a scope-specific error. + def test_raises_on_non_string_content(self): + """Content that is neither a string nor None (e.g. an int) raises ValueError with its type.""" + with pytest.raises(ValueError, match="Expected string content, got int"): + _parse_chat_completion({"choices": [{"message": {"content": 123}}]}) + + def test_parses_tool_calls_when_content_none(self): + """content=None with tool_calls parses the calls and normalizes content to ''. - OpenAI returns this shape for tool_choice='required'. Tool-call support is out of - scope for this PR series; the error message should make that clear rather than - suggesting malformed data. + OpenAI returns this shape for tool_choice='required'; arguments arrive as a JSON + string on the wire and are normalized into a dict. """ - with pytest.raises(ValueError, match="Tool-call-only responses are not yet supported"): - _parse_chat_completion( - { - "choices": [ - { - "message": { - "role": "assistant", - "content": None, - "tool_calls": [{"id": "x", "type": "function", "function": {"name": "f"}}], - } + result = _parse_chat_completion( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "x", "type": "function", "function": {"name": "f", "arguments": '{"a": 1}'}} + ], + }, + "finish_reason": "tool_calls", + } + ] + } + ) + assert result.content == "" + assert result.finish_reason == "tool_calls" + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].id == "x" + assert result.tool_calls[0].function.name == "f" + assert result.tool_calls[0].function.arguments == {"a": 1} + + def test_parses_tool_calls_alongside_text_content(self): + """A response may carry both text content and tool calls; both are surfaced.""" + result = _parse_chat_completion( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": "Let me look that up.", + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "search", "arguments": "{}"}} + ], } - ] - } - ) + } + ] + } + ) + assert result.content == "Let me look that up." + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "search" + assert result.tool_calls[0].function.arguments == {} + + def test_parses_parallel_tool_calls(self): + """Multiple tool calls in one response are all parsed into the list, in order.""" + result = _parse_chat_completion( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "get_time", "arguments": '{"city": "Paris"}'}, + }, + ], + }, + "finish_reason": "tool_calls", + } + ] + } + ) + assert [tc.function.name for tc in result.tool_calls] == ["get_weather", "get_time"] + assert [tc.id for tc in result.tool_calls] == ["c1", "c2"] + + def test_reasoning_preserved_alongside_tool_calls(self): + """NIM-style responses carry reasoning_content together with tool calls.""" + result = _parse_chat_completion( + { + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "The user wants the weather.", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + "finish_reason": "tool_calls", + } + ] + } + ) + assert result.reasoning == "The user wants the weather." + assert result.content == "" + assert len(result.tool_calls) == 1 + + def test_text_response_has_no_tool_calls(self): + """A normal text response leaves tool_calls as None.""" + result = _parse_chat_completion({"choices": [{"message": {"content": "hi"}}]}) + assert result.tool_calls is None class TestParseChatCompletionChunk: From 81df83599b8e0b7158107fda7c213182bac5eff2 Mon Sep 17 00:00:00 2001 From: Tim Gasser <200644301+tgasser-nv@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:22:04 -0500 Subject: [PATCH 06/21] fix(iorails): Apply inference-time llm_params on top of Model.parameters in ModelEngine (#2020) --- nemoguardrails/guardrails/engine_registry.py | 19 +- nemoguardrails/guardrails/model_engine.py | 50 ++++- tests/guardrails/test_engine_registry.py | 208 +++++++++++++++++++ tests/guardrails/test_model_engine.py | 87 ++++++++ 4 files changed, 359 insertions(+), 5 deletions(-) diff --git a/nemoguardrails/guardrails/engine_registry.py b/nemoguardrails/guardrails/engine_registry.py index d68bb21d8f..3fd6b2c249 100644 --- a/nemoguardrails/guardrails/engine_registry.py +++ b/nemoguardrails/guardrails/engine_registry.py @@ -195,6 +195,10 @@ async def model_call(self, model_type: str, messages: list[dict], **kwargs: Any) provider_name = engine.model_config.engine or "unknown" operation_name = "chat" + # Merge the model's config parameters with per-call kwargs (GenerationOptions.llm_params) + # Per-call kwargs have priority. + merged_params = {**engine.body_param_defaults, **kwargs} + # Compose: span (always created — no-op when tracer is None) and # duration metric (only when metrics enabled). Token usage is # emitted after the call returns since it depends on @@ -208,9 +212,9 @@ async def model_call(self, model_type: str, messages: list[dict], **kwargs: Any) with llm_call_span(self._tracer, engine.model_name, provider_name, operation_name) as span: # Request params are known before the call, so set them first — # they land on the span even if the call raises. - set_llm_request_attributes(span, kwargs) + set_llm_request_attributes(span, merged_params) with duration_ctx: - result = await engine.chat_completion(messages, **kwargs) + result = await engine.chat_completion(messages, **merged_params) # Set response/usage and content attrs inside the span context so # the helpers see the live LLM CLIENT span and the attributes land # before it closes. Both are skipped on exception, which never @@ -270,6 +274,13 @@ async def stream_model_call( provider_name = engine.model_config.engine or "unknown" operation_name = "chat" + # Merge the model's configured parameter defaults with the per-call + # kwargs (per-call wins), above set_llm_request_attributes, so the span + # reflects the request body. Excluding "stream"/"stream_options" from + # body_param_defaults also prevents a duplicate-keyword TypeError when + # stream_call() passes its own stream=True into _prepare_request(). + merged_params = {**engine.body_param_defaults, **kwargs} + # Capture the latest non-None response fields from the stream so we # can set the LLM span's response/usage attrs and emit the token # metric after the stream completes. Providers spread these across @@ -295,7 +306,7 @@ async def stream_model_call( with llm_call_span(self._tracer, engine.model_name, provider_name, operation_name) as span: # Set request params + stream=True before the first chunk so they # land on the span even if the stream errors mid-flight. - set_llm_request_attributes(span, kwargs, stream=True) + set_llm_request_attributes(span, merged_params, stream=True) with duration_ctx: # Gate timing-state setup on ``_metrics_enabled`` so the # cold path skips ``time.monotonic()`` and the per-chunk @@ -304,7 +315,7 @@ async def stream_model_call( # — it's never read in that branch. t0 = time.monotonic() if self._metrics_enabled else 0.0 last_chunk_time: Optional[float] = None - async for chunk in engine.stream_chat_completion(messages, **kwargs): + async for chunk in engine.stream_chat_completion(messages, **merged_params): if self._metrics_enabled: # Per OTEL semconv, "first chunk" / "output chunk" # mean content-bearing chunks — gate on diff --git a/nemoguardrails/guardrails/model_engine.py b/nemoguardrails/guardrails/model_engine.py index 307ef7b4da..67db82fa9c 100644 --- a/nemoguardrails/guardrails/model_engine.py +++ b/nemoguardrails/guardrails/model_engine.py @@ -24,7 +24,8 @@ import logging import os import time -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping +from types import MappingProxyType from typing import Any, NamedTuple, Optional, cast import aiohttp @@ -51,6 +52,46 @@ _CHAT_COMPLETIONS_ENDPOINT = "/v1/chat/completions" +# Parameter keys the engine reserves and handles itself, so they are NOT +# forwarded into the /v1/chat/completions request body. Everything else in +# a model's ``parameters`` config block becomes a per-request default via +# ``ModelEngine.body_param_defaults``. +_RESERVED_LLM_PARAMETERS = frozenset( + { + # transport / retry — consumed by ModelEngine.__init__ and + # _resolve_base_url, never part of the request body + "base_url", + "timeout", + "timeout_connect", + "max_attempts", + # secret — carried in the Authorization header, never echoed into the body + "api_key", + # model identity / positional — set explicitly by _prepare_request; + # would collide with the "model" body field set there. After Model + # validation these never appear in parameters, but exclude them + # defensively against direct construction. + "model", + "model_name", + "messages", + # streaming control — owned by the engine. "stream" in particular + # collides with the explicit stream=True kwarg that stream_call() + # passes into _prepare_request(), which would raise TypeError on a + # duplicate keyword argument. + "stream", + # Can't set Model-level `stream_options` because the same model can + # be used in streaming or non-streaming mode. Defer to inference-time + # `llm_params`. + "stream_options", + # client-only options — these configure the OpenAI-compatible client + # (constructor kwargs), not the chat-completion request body. IORails + # doesn't wire the shared client yet; reserve them so they're never + # forwarded as body fields, leaving proper client support to a future + # refactor. + "default_headers", + "default_query", + } +) + class _RequestParams(NamedTuple): """Pre-built parameters for an HTTP request to the completions endpoint.""" @@ -201,6 +242,13 @@ def __init__(self, model_config: Model) -> None: max_attempts=int(params.get("max_attempts") or DEFAULT_MAX_ATTEMPTS), ) + # Default `llm_params` used on inference are the subset of Model.parameters after + # filtering out keys in _RESERVED_LLM_PARAMETERS. Exposed as a read-only + # MappingProxyType view so callers can't mutate the shared per-engine defaults. + self.body_param_defaults: Mapping[str, Any] = MappingProxyType( + {key: value for key, value in params.items() if key not in _RESERVED_LLM_PARAMETERS} + ) + def _resolve_base_url(self) -> str: """Resolve the base URL from model parameters or engine type. diff --git a/tests/guardrails/test_engine_registry.py b/tests/guardrails/test_engine_registry.py index 6397e25200..668c3a83f4 100644 --- a/tests/guardrails/test_engine_registry.py +++ b/tests/guardrails/test_engine_registry.py @@ -154,6 +154,28 @@ async def _readline(): return mock_response +@patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) +def _registry_with_main_params(parameters: dict, tracer): + """Build an EngineRegistry whose ``main`` model carries ``parameters``, + wired to ``tracer`` so model_call / stream_model_call produce real spans we + can read back. Used by the parameter-defaults merge tests, which need a + model with non-empty ``parameters`` (the shared NEMOGUARDS_CONFIG models + have none).""" + config = RailsConfig.from_content( + config={ + "models": [ + { + "type": "main", + "engine": "nim", + "model": "meta/llama-3.3-70b-instruct", + "parameters": parameters, + } + ] + } + ) + return EngineRegistry(config.models, config.rails.config, tracer=tracer) + + class TestEngineRegistryInit: """Test EngineRegistry creates engines from config.""" @@ -505,6 +527,192 @@ async def test_request_attributes_present_on_error(self, manager_with_tracer, sp assert attrs["error.type"] == "RuntimeError" +class TestEngineRegistryParameterDefaults: + """``model_call`` / ``stream_model_call`` merge ModelEngine.body_param_defaults + (the model's ``parameters`` config minus transport/secret/streaming keys) + under the per-call kwargs. Both the request body and the gen_ai.request.* + span attrs reflect the defaults, with per-call llm_params overriding.""" + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_model_call_applies_config_defaults(self, span_exporter): + """No per-call kwargs → the model's parameter defaults populate both the + request body and the request span attrs.""" + tracer, exporter = span_exporter + registry = _registry_with_main_params({"temperature": 0.7, "max_tokens": 256}, tracer) + engine = registry._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock(return_value=LLMResponse(content="ok")) + + await registry.model_call("main", [{"role": "user", "content": "hi"}]) + + body = engine.chat_completion.call_args[1] + assert body["temperature"] == 0.7 + assert body["max_tokens"] == 256 + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == 0.7 + assert attrs["gen_ai.request.max_tokens"] == 256 + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_model_call_per_call_kwargs_override_defaults(self, span_exporter): + """A per-call kwarg overrides the config default for that key; the other + defaults are retained, in both body and span.""" + tracer, exporter = span_exporter + registry = _registry_with_main_params({"temperature": 0.7, "max_tokens": 256}, tracer) + engine = registry._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock(return_value=LLMResponse(content="ok")) + + await registry.model_call("main", [{"role": "user", "content": "hi"}], temperature=0.1) + + body = engine.chat_completion.call_args[1] + assert body["temperature"] == 0.1 # per-call override wins + assert body["max_tokens"] == 256 # config default retained + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == 0.1 + assert attrs["gen_ai.request.max_tokens"] == 256 + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_llm_params_take_precedence_over_config_parameters(self, span_exporter): + """Precedence guard: when the same key is set in BOTH the static + Model.parameters config AND the per-call llm_params, the per-call value + must win in the request body actually sent to the engine. Reversing the + merge order ({**kwargs, **defaults}, config winning) flips the sent + temperature back to the config value and fails this test.""" + config_temperature = 0.7 + per_call_temperature = 0.1 + tracer, exporter = span_exporter + registry = _registry_with_main_params({"temperature": config_temperature}, tracer) + engine = registry._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock(return_value=LLMResponse(content="ok")) + + await registry.model_call("main", [{"role": "user", "content": "hi"}], temperature=per_call_temperature) + + sent_body = engine.chat_completion.call_args[1] + assert sent_body["temperature"] == per_call_temperature + assert sent_body["temperature"] != config_temperature # the static config default must not win + # The span reflects the same value that was sent. + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == per_call_temperature + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_model_call_excludes_non_body_keys(self, span_exporter): + """Transport (base_url/timeout) and streaming-control (stream) keys in + parameters never reach the request body; only the sampling param does.""" + tracer, _ = span_exporter + registry = _registry_with_main_params( + {"base_url": "https://custom.example.com", "timeout": 5, "stream": True, "temperature": 0.5}, + tracer, + ) + engine = registry._get_engine("main", ModelEngine) + engine.chat_completion = AsyncMock(return_value=LLMResponse(content="ok")) + + await registry.model_call("main", [{"role": "user", "content": "hi"}]) + + body = engine.chat_completion.call_args[1] + assert body == {"temperature": 0.5} + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_stream_model_call_applies_defaults_and_override(self, span_exporter): + """Streaming path merges the same way: defaults populate the body + (captured off the engine call) and the span attrs (with stream=True), + and a per-call kwarg overrides its default.""" + tracer, exporter = span_exporter + registry = _registry_with_main_params({"temperature": 0.7, "max_tokens": 256}, tracer) + engine = registry._get_engine("main", ModelEngine) + + captured: dict = {} + + async def _capturing_stream(messages, **kwargs): # noqa: ARG001 (signature dictated by ModelEngine) + captured.update(kwargs) + yield LLMResponseChunk(delta_content="hi", finish_reason="stop") + + engine.stream_chat_completion = _capturing_stream + + async for _ in registry.stream_model_call("main", [{"role": "user", "content": "hi"}], temperature=0.1): + pass + + assert captured["temperature"] == 0.1 # per-call override wins + assert captured["max_tokens"] == 256 # config default retained + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.stream"] is True + assert attrs["gen_ai.request.temperature"] == 0.1 + assert attrs["gen_ai.request.max_tokens"] == 256 + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_stream_llm_params_take_precedence_over_config_parameters(self, span_exporter): + """Streaming precedence guard: when the same key is set in BOTH the + static Model.parameters config AND the per-call llm_params, the per-call + value must win in what stream_model_call forwards to the engine. + Reversing the merge order ({**kwargs, **defaults}, config winning) flips + the streamed temperature back to the config value and fails this test.""" + config_temperature = 0.7 + per_call_temperature = 0.1 + tracer, exporter = span_exporter + registry = _registry_with_main_params({"temperature": config_temperature}, tracer) + engine = registry._get_engine("main", ModelEngine) + + captured: dict = {} + + async def _capturing_stream(messages, **kwargs): # noqa: ARG001 (signature dictated by ModelEngine) + captured.update(kwargs) + yield LLMResponseChunk(delta_content="hi", finish_reason="stop") + + engine.stream_chat_completion = _capturing_stream + + async for _ in registry.stream_model_call( + "main", [{"role": "user", "content": "hi"}], temperature=per_call_temperature + ): + pass + + assert captured["temperature"] == per_call_temperature + assert captured["temperature"] != config_temperature # the static config default must not win + # The span reflects the same value that was forwarded. + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == per_call_temperature + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "test-key"}) + @pytest.mark.asyncio + async def test_stream_model_call_with_stream_param_does_not_raise_type_error(self, span_exporter): + """A model whose parameters include ``stream`` drives the real + stream_call/_prepare_request path without a duplicate-keyword TypeError + (stream is excluded from body_param_defaults). Only the sampling param + reaches the sent body; transport/streaming keys are dropped.""" + tracer, exporter = span_exporter + registry = _registry_with_main_params( + {"stream": True, "base_url": "https://custom.example.com", "temperature": 0.5}, tracer + ) + engine = registry._get_engine("main", ModelEngine) + engine._client = AsyncMock() + engine._client.post = MagicMock( + return_value=_mock_sse_response( + [ + { + "id": "chatcmpl-stream", + "model": "meta/llama-3.3-70b-instruct", + "choices": [{"delta": {"content": "hi"}, "finish_reason": "stop"}], + }, + ] + ) + ) + engine._running = True + + # Must not raise TypeError("got multiple values for keyword argument 'stream'"). + async for _ in registry.stream_model_call("main", [{"role": "user", "content": "hi"}]): + pass + + sent_body = engine._client.post.call_args.kwargs["json"] + assert sent_body["temperature"] == 0.5 + assert sent_body["stream"] is True # set explicitly by stream_call, not from parameters + assert "base_url" not in sent_body + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs["gen_ai.request.temperature"] == 0.5 + assert attrs["gen_ai.request.stream"] is True + + class TestEngineRegistryStartErrors: """Test EngineRegistry start() error handling and rollback.""" diff --git a/tests/guardrails/test_model_engine.py b/tests/guardrails/test_model_engine.py index 02e4b40874..ce9c2b15d0 100644 --- a/tests/guardrails/test_model_engine.py +++ b/tests/guardrails/test_model_engine.py @@ -328,6 +328,93 @@ def test_client_initially_none(self): assert engine._client is None +class TestModelEngineBodyParamDefaults: + """Test ModelEngine.body_param_defaults: sampling params from a model's + ``parameters`` config become per-request body defaults, while transport, + secret, identity, and streaming-control keys are excluded.""" + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_keeps_sampling_params(self): + """Sampling/body params (temperature, max_tokens, seed, top_p, ...) all + pass through to body_param_defaults unchanged.""" + engine = ModelEngine(_make_model(parameters={"temperature": 0.3, "max_tokens": 256, "seed": 42, "top_p": 0.9})) + assert engine.body_param_defaults == { + "temperature": 0.3, + "max_tokens": 256, + "seed": 42, + "top_p": 0.9, + } + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_excludes_transport_and_retry_keys(self): + """Transport/retry keys consumed by __init__ and _resolve_base_url are + not echoed into the body; a sampling param alongside them is kept.""" + engine = ModelEngine( + _make_model( + engine="nim", + parameters={ + "base_url": "https://custom.example.com", + "timeout": 120, + "timeout_connect": 30, + "max_attempts": 5, + "temperature": 0.5, + }, + ) + ) + assert engine.body_param_defaults == {"temperature": 0.5} + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_excludes_secret_and_streaming_keys(self): + """api_key (secret, header-only) and stream/stream_options (engine-owned) + never reach the body defaults.""" + engine = ModelEngine( + _make_model( + parameters={ + "api_key": "sk-should-not-leak", + "stream": True, + "stream_options": {"include_usage": True}, + "max_tokens": 64, + } + ) + ) + assert engine.body_param_defaults == {"max_tokens": 64} + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_excludes_client_only_keys(self): + """default_headers / default_query configure the OpenAI-compatible + client, not the chat-completion body, so they never reach the body + defaults; a sampling param alongside them is kept.""" + engine = ModelEngine( + _make_model( + parameters={ + "default_headers": {"X-Tenant": "acme"}, + "default_query": {"api-version": "2024-02-01"}, + "temperature": 0.5, + } + ) + ) + assert engine.body_param_defaults == {"temperature": 0.5} + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_excludes_identity_keys_defensively(self): + """model / model_name / messages are stripped even when present in + parameters. The Model validator normally lifts model/model_name into + the model field, so these only leak via direct construction — mutate + parameters after build to simulate that path.""" + model = _make_model(parameters={"temperature": 0.2}) + model.parameters.update( + {"model": "shadow", "model_name": "shadow", "messages": [{"role": "user", "content": "x"}]} + ) + engine = ModelEngine(model) + assert engine.body_param_defaults == {"temperature": 0.2} + + @patch.dict("os.environ", {"NVIDIA_API_KEY": "key"}) + def test_empty_parameters_yields_empty_defaults(self): + """A model with no parameters has empty body_param_defaults.""" + engine = ModelEngine(_make_model(parameters={})) + assert engine.body_param_defaults == {} + + class TestModelEngineLifecycle: """Test the ModelEngine start() and stop() client lifecycle.""" From fb73f41b0fa2ef6c50c90d76f8b20533e4c9a51e Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Mon, 15 Jun 2026 10:27:25 -0700 Subject: [PATCH 07/21] docs: fix remaining links after migration and rm temporary local link checker (#2018) --- .coderabbit.yaml | 9 +- .github/workflows/docs-links-pr.yaml | 146 ---- Makefile | 6 +- docs/about/release-notes.mdx | 24 +- .../colang-2/getting-started/dialog-rails.mdx | 4 +- .../colang-2/getting-started/hello-world.mdx | 4 +- .../colang/colang-2/getting-started/index.mdx | 18 +- .../colang-2/getting-started/input-rails.mdx | 2 +- .../getting-started/multimodal-rails.mdx | 2 +- .../recommended-next-steps.mdx | 2 +- .../configure-rails/colang/colang-2/index.mdx | 8 +- .../colang-2/language-reference/index.mdx | 44 +- .../language-reference/make-use-of-llms.mdx | 12 +- .../the-standard-library.mdx | 14 +- docs/configure-rails/colang/index.mdx | 4 +- .../community/ai-defense.mdx | 2 +- .../guardrail-catalog/community/clavata.mdx | 4 +- .../guardrail-catalog/community/cleanlab.mdx | 2 +- .../guardrail-catalog/community/fiddler.mdx | 8 +- .../community/llama-guard.mdx | 2 +- .../community/patronus-evaluate-api.mdx | 2 +- .../guardrail-catalog/content-safety.mdx | 6 +- .../guardrail-catalog/third-party.mdx | 6 +- .../guardrail-catalog/topic-control.mdx | 4 +- docs/evaluation/evaluate-guardrails.mdx | 6 +- .../evaluation/llm-vulnerability-scanning.mdx | 8 +- .../jailbreak-detection-heuristics.mdx | 2 +- .../langchain/agent-middleware.mdx | 2 +- docs/resources/research.mdx | 2 +- docs/telemetry.mdx | 2 +- fern/docs.yml | 244 +++++- .../colang/v1_0/lang/comd_parser.py | 14 +- nemoguardrails/tracing/constants.py | 2 +- nemoguardrails/utils.py | 5 +- scripts/check-docs-links.sh | 719 ------------------ tests/test_docs_links.py | 184 ----- 36 files changed, 320 insertions(+), 1205 deletions(-) delete mode 100644 .github/workflows/docs-links-pr.yaml delete mode 100755 scripts/check-docs-links.sh delete mode 100644 tests/test_docs_links.py diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 7a20596e2a..53b1ddd1f2 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -46,6 +46,13 @@ reviews: # Default: true review_status: false # current: true path_instructions: + - path: "fern/docs.yml" + instructions: | + Redirect destinations for documentation pages must use the rendered Fern route from docs/index.yml, not the source file path. + In docs/index.yml, the rendered URL comes from the section/page `slug` hierarchy; the `path` field only points to the source MDX file. + For example, the source path docs/configure-rails/colang/colang-2/language-reference/introduction.mdx renders under /configure-guardrails/colang/colang-2/language-reference/introduction because the top-level Configure Guardrails section has slug `configure-guardrails`. + Do not suggest replacing valid rendered routes such as /configure-guardrails/... with source-path-derived routes such as /configure-rails/.... + Before flagging a redirect destination as wrong, verify the route by tracing the relevant entries in docs/index.yml. - path: "docs/**/*.mdx" instructions: | Internal absolute documentation links in Fern MDX pages must use the rendered Fern route, not the source file path. @@ -54,7 +61,7 @@ reviews: - docs/configure-rails/colang/index.mdx renders at /configure-guardrails/colang, not /configure-rails/colang. - docs/run-rails/index.mdx renders at /run-guardrailed-inference/run-rails, not /run-rails. - docs/getting-started/installation-guide.mdx renders at /get-started/installation-guide, not /getting-started/installation-guide. - Before flagging or suggesting fixes for internal docs links, verify them with scripts/check-docs-links.sh or docs/index.yml. + Before flagging or suggesting fixes for internal docs links, verify them with docs/index.yml. Do not suggest changing valid Fern routes back to source-path-style links. - path: "docs/**/*.ipynb" instructions: | diff --git a/.github/workflows/docs-links-pr.yaml b/.github/workflows/docs-links-pr.yaml deleted file mode 100644 index 0c67ca07d4..0000000000 --- a/.github/workflows/docs-links-pr.yaml +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Docs / Link Check - -on: - pull_request: - branches: [develop] - types: [opened, reopened, synchronize] - paths: - - "**/*.md" - - "**/*.mdx" - - "docs/index.yml" - - ".github/workflows/docs-links-pr.yaml" - - "scripts/check-docs-links.sh" - -permissions: - contents: read - -jobs: - markdown-links: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "22" - - - name: Determine changed documentation files - id: changed - shell: bash - run: | - set -euo pipefail - base="${{ github.event.pull_request.base.sha }}" - head="${{ github.event.pull_request.head.sha }}" - mapfile -t doc_files < <( - # Fern validates MDX routes during the docs build. This raw - # Markdown job only checks repository-local .md links. - git diff --name-only --diff-filter=ACMR "$base" "$head" -- \ - '*.md' \ - ':(exclude)node_modules/**' \ - ':(exclude)dist/**' \ - ':(exclude)vendor/**' \ - ':(exclude)build/**' \ - | LC_ALL=C sort -u - ) - - if [[ "${#doc_files[@]}" -eq 0 ]]; then - echo "has_files=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "has_files=true" >> "$GITHUB_OUTPUT" - { - echo "files<> "$GITHUB_OUTPUT" - - - name: Run markdown link checker - if: steps.changed.outputs.has_files == 'true' - shell: bash - run: | - set -euo pipefail - mapfile -t doc_files <<< "${{ steps.changed.outputs.files }}" - bash scripts/check-docs-links.sh --local-only "${doc_files[@]}" - - fern-links: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "22" - - - name: Determine changed Fern documentation files - id: changed - shell: bash - run: | - set -euo pipefail - base="${{ github.event.pull_request.base.sha }}" - head="${{ github.event.pull_request.head.sha }}" - mapfile -t changed_files < <( - git diff --name-only --diff-filter=ACMR "$base" "$head" -- \ - '*.mdx' \ - 'docs/index.yml' \ - 'scripts/check-docs-links.sh' \ - ':(exclude)docs/_static/python-sdk-reference/**' \ - ':(exclude)node_modules/**' \ - ':(exclude)dist/**' \ - ':(exclude)vendor/**' \ - ':(exclude)build/**' \ - | LC_ALL=C sort -u - ) - - full_scan=false - fern_files=() - for file in "${changed_files[@]}"; do - case "$file" in - docs/index.yml | scripts/check-docs-links.sh) - full_scan=true - ;; - *.mdx) - fern_files+=("$file") - ;; - esac - done - - if [[ "$full_scan" == "false" && "${#fern_files[@]}" -eq 0 ]]; then - echo "has_files=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "has_files=true" >> "$GITHUB_OUTPUT" - echo "full_scan=$full_scan" >> "$GITHUB_OUTPUT" - { - echo "files<> "$GITHUB_OUTPUT" - - - name: Run Fern route link checker - if: steps.changed.outputs.has_files == 'true' - shell: bash - run: | - set -euo pipefail - if [[ "${{ steps.changed.outputs.full_scan }}" == "true" ]]; then - bash scripts/check-docs-links.sh --local-only - else - mapfile -t fern_files <<< "${{ steps.changed.outputs.files }}" - bash scripts/check-docs-links.sh --local-only "${fern_files[@]}" - fi diff --git a/Makefile b/Makefile index 82349f1f1b..2e2621ea2d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: help .PHONY: test test-parallel test-serial test-benchmark test-watch test-coverage test-profile warm-fastembed-cache -.PHONY: docs-fern docs-fern-strict docs-fern-live docs-fern-preview-watch docs-fern-generate-sdk docs-fern-fix-empty-links docs-check-links docs-check-redirects docs-fern-publish-staging docs-fern-publish-public +.PHONY: docs-fern docs-fern-strict docs-fern-live docs-fern-preview-watch docs-fern-generate-sdk docs-fern-fix-empty-links docs-check-redirects docs-fern-publish-staging docs-fern-publish-public .PHONY: pre-commit .DEFAULT_GOAL := help @@ -70,9 +70,6 @@ docs-fern-generate-sdk: docs-fern-fix-empty-links: node scripts/fix-empty-fern-links.mjs -docs-check-links: - bash scripts/check-docs-links.sh --local-only - docs-check-redirects: cd docs && poetry run python scripts/validate_redirects.py @@ -109,7 +106,6 @@ help: ' docs-fern-preview-watch Watch and publish Fern preview for the current branch' \ ' docs-fern-generate-sdk Regenerate Python SDK reference pages with Fern' \ ' docs-fern-fix-empty-links Replace empty Markdown links with titles from Fern navigation' \ - ' docs-check-links Validate Markdown and MDX links locally' \ ' docs-check-redirects Validate docs redirects' \ '' \ 'Maintenance:' \ diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index a169842316..bdf4c6f46f 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -6,8 +6,9 @@ sidebar-title: "Release Notes" description: "Review new features, breaking changes, and fixed issues for each release." keywords: ["nemo guardrails changelog", "release notes", "version history"] content: - type: "reference" + type: "reference" --- + The following sections summarize and highlight the changes for each release. For a complete record of changes in a release, refer to the [CHANGELOG.md](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/CHANGELOG.md) in the GitHub repository. @@ -118,14 +119,13 @@ For a complete record of changes in a release, refer to the --- -## Previous Release Notes - -- [0.21.0](https://docs.nvidia.com/nemo/guardrails/0.21.0/release-notes.html) -- [0.20.0](https://docs.nvidia.com/nemo/guardrails/0.20.0/release-notes.html) -- [0.19.0](https://docs.nvidia.com/nemo/guardrails/0.19.0/release-notes.html) -- [0.18.0](https://docs.nvidia.com/nemo/guardrails/0.18.0/release-notes.html) -- [0.17.0](https://docs.nvidia.com/nemo/guardrails/0.17.0/release-notes.html) -- [0.16.0](https://docs.nvidia.com/nemo/guardrails/0.16.0/release-notes.html) -- [0.15.0](https://docs.nvidia.com/nemo/guardrails/0.15.0/release-notes.html) -- [0.14.1](https://docs.nvidia.com/nemo/guardrails/0.14.1/release-notes.html) -- [0.14.0](https://docs.nvidia.com/nemo/guardrails/0.14.0/release-notes.html) +## Previous Releases + +- [0.21.0](https://archive.docs.nvidia.com/nemo/guardrails/0.21.0/index.html) +- [0.20.0](https://archive.docs.nvidia.com/nemo/guardrails/0.20.0/index.html) +- [0.19.0](https://archive.docs.nvidia.com/nemo/guardrails/0.19.0/index.html) +- [0.18.0](https://archive.docs.nvidia.com/nemo/guardrails/0.18.0/index.html) +- [0.17.0](https://archive.docs.nvidia.com/nemo/guardrails/0.17.0/index.html) +- [0.16.0](https://archive.docs.nvidia.com/nemo/guardrails/0.16.0/index.html) +- [0.15.0](https://archive.docs.nvidia.com/nemo/guardrails/0.15.0/index.html) +- [0.14.1](https://archive.docs.nvidia.com/nemo/guardrails/0.14.1/index.html) diff --git a/docs/configure-rails/colang/colang-2/getting-started/dialog-rails.mdx b/docs/configure-rails/colang/colang-2/getting-started/dialog-rails.mdx index 8ff43a77f5..75df1d95fd 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/dialog-rails.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/dialog-rails.mdx @@ -42,14 +42,14 @@ flow bot express greeting ``` -The recommended practice is to use past tense for matching external actions, like the user saying something, and present for bot actions that must be executed. See [Defining Flows](../language-reference/defining-flows#flow-naming-conventions) for more details. +The recommended practice is to use past tense for matching external actions, like the user saying something, and present for bot actions that must be executed. See [Defining Flows](/configure-guardrails/colang/colang-2/language-reference/defining-flows#flow-naming-conventions) for more details. ## LLM Integration While the example above has more structure, it is still rigid in the sense that it only works with the exact inputs "hi" and "hello". -To enable the use of the LLM to drive the interaction for inputs that are not matched exactly by flows, you have to *activate* the `llm continuation` flow, which is part of the `llm` module in the [Colang Standard Library (CSL)](../language-reference/the-standard-library#the-standard-library). +To enable the use of the LLM to drive the interaction for inputs that are not matched exactly by flows, you have to *activate* the `llm continuation` flow, which is part of the `llm` module in the [Colang Standard Library (CSL)](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library). **examples/v2_x/tutorial/hello_world_3/main.co** diff --git a/docs/configure-rails/colang/colang-2/getting-started/hello-world.mdx b/docs/configure-rails/colang/colang-2/getting-started/hello-world.mdx index c3065cb0cd..991feaea9e 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/hello-world.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/hello-world.mdx @@ -27,7 +27,7 @@ flow main ``` -You can find the full example from this guide [here](../../../../../examples/v2_x/tutorial/hello_world_1). +You can find the full example from this guide [here](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/examples/v2_x/tutorial/hello_world_1). The achieve this, the `main` flow uses two pre-defined flows: @@ -39,7 +39,7 @@ The two flows are located in the `core` module, included in the Colang Standard -For more details, check out the [Colang Standard Library (CSL)](../language-reference/the-standard-library#the-standard-library). +For more details, check out the [Colang Standard Library (CSL)](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library). ## Testing diff --git a/docs/configure-rails/colang/colang-2/getting-started/index.mdx b/docs/configure-rails/colang/colang-2/getting-started/index.mdx index 8cb966bc03..c93bb39df1 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/index.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/index.mdx @@ -26,11 +26,11 @@ models: engine: openai model: gpt-4-turbo ``` -The above config sets the Colang version to "2.x" (this is needed since "1.0" is currently the default) and the LLM engine to OpenAI's `gpt-4-turbo`. Make sure to set the required API access key as an environment variable (e.g. OPENAI_API_KEY for OpenAI API). See section [Supported Models](../language-reference/make-use-of-llms#make-use-of-llms-supported-models) for all supported models. +The above config sets the Colang version to "2.x" (this is needed since "1.0" is currently the default) and the LLM engine to OpenAI's `gpt-4-turbo`. Make sure to set the required API access key as an environment variable (e.g. OPENAI_API_KEY for OpenAI API). See section [Supported Models](/configure-guardrails/colang/colang-2/language-reference/make-use-of-llms#make-use-of-llms-supported-models) for all supported models. -Check the section [Development and Debugging](../language-reference/development-and-debugging#development-and-debugging) for how you can install Colang syntax highlighting to make editing Colang scripts easier. +Check the section [Development and Debugging](/configure-guardrails/colang/colang-2/language-reference/development-and-debugging#development-and-debugging) for how you can install Colang syntax highlighting to make editing Colang scripts easier. ## Terminology @@ -40,43 +40,43 @@ At a high level, Colang adopts as much as possible from the Python terminology. ## Guides - + Create the first Colang 2.0 flow. - + Add dialog rails with LLM integration. - + Use multimodal events and actions. - + Add input rails to inspect and block user input. - + Build a simple interaction loop. - + Let the LLM generate flow continuations. - + Choose what to learn after the getting started guide. diff --git a/docs/configure-rails/colang/colang-2/getting-started/input-rails.mdx b/docs/configure-rails/colang/colang-2/getting-started/input-rails.mdx index de8dbc99da..a56778fdbd 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/input-rails.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/input-rails.mdx @@ -19,7 +19,7 @@ This section explains how to create *input rails* in Colang 2.0. To activate input rails in Colang 2.0, you must: -1. Import the `guardrails` module from the [Colang Standard Library (CSL)](../language-reference/the-standard-library#the-standard-library). +1. Import the `guardrails` module from the [Colang Standard Library (CSL)](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library). 2. Define a flow called `input rails`, which takes a single parameter called `$input_text`. In the example below, the `input rails` flow calls another flow named `check user message` which prompts the LLM to check the input. diff --git a/docs/configure-rails/colang/colang-2/getting-started/multimodal-rails.mdx b/docs/configure-rails/colang/colang-2/getting-started/multimodal-rails.mdx index ceccb059da..1cbc37e0c1 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/multimodal-rails.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/multimodal-rails.mdx @@ -19,7 +19,7 @@ The example below shows how you can control the greeting behavior of an interact -The [Colang Standard Library (CSL)](../language-reference/the-standard-library#the-standard-library) includes an `avatars` module with flows for multimodal events and actions to implement interactive avatars use cases. +The [Colang Standard Library (CSL)](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library) includes an `avatars` module with flows for multimodal events and actions to implement interactive avatars use cases. **examples/v2_x/tutorial/multi_modal/main.co** diff --git a/docs/configure-rails/colang/colang-2/getting-started/recommended-next-steps.mdx b/docs/configure-rails/colang/colang-2/getting-started/recommended-next-steps.mdx index a868545b10..0f7c2061f3 100644 --- a/docs/configure-rails/colang/colang-2/getting-started/recommended-next-steps.mdx +++ b/docs/configure-rails/colang/colang-2/getting-started/recommended-next-steps.mdx @@ -12,6 +12,6 @@ The Colang 2.0 getting started guide introduces you to a basic [Hello World](hel This only scratches the surface of what can be achieved with Colang 2.0. -If you are an experienced developer and want to learn about the syntax and various features in details, we recommend going through the [Language Reference](../language-reference/index#colang_2_language_reference) documentation. +If you are an experienced developer and want to learn about the syntax and various features in details, we recommend going through the [Language Reference](/configure-guardrails/colang/colang-2/language-reference#colang_2_language_reference) documentation. Version `0.10.0` of NeMo Guardrails will add more examples including RAG and agents. Also it will bring support for the Guardrails Library, which will enable you to use any of the existing guardrails similar to Colang 1.0. diff --git a/docs/configure-rails/colang/colang-2/index.mdx b/docs/configure-rails/colang/colang-2/index.mdx index c89963bee0..c04e3c62b8 100644 --- a/docs/configure-rails/colang/colang-2/index.mdx +++ b/docs/configure-rails/colang/colang-2/index.mdx @@ -12,25 +12,25 @@ Use this guide to review what changed from Colang 1.0, migrate existing configur - + Review the main language and runtime changes in Colang 2.0. - + Convert Colang 1.0 configurations to Colang 2.x. - + Work through the Colang 2.0 tutorial sequence. - + Explore syntax and standard library reference material. diff --git a/docs/configure-rails/colang/colang-2/language-reference/index.mdx b/docs/configure-rails/colang/colang-2/language-reference/index.mdx index 5528e127e6..9130c90dc4 100644 --- a/docs/configure-rails/colang/colang-2/language-reference/index.mdx +++ b/docs/configure-rails/colang/colang-2/language-reference/index.mdx @@ -10,67 +10,67 @@ description: "Reference documentation for Colang 2.0 syntax, events, actions, fl - + Understand Colang 2.0 concepts and basic syntax. - + Learn how Colang generates and matches events. - + Use actions and action events in Colang. - + Define, start, await, and compose flows. - + Use variables and expressions in Colang. - + Use branching, loops, and return or abort statements. - + Use the Colang Standard Library modules. - + Configure and use LLM-related features. - + Explore advanced flow behavior and conflict resolution. - + Call custom Python actions from Colang. - + Debug Colang flows and inspect runtime state. @@ -80,14 +80,14 @@ Debug Colang flows and inspect runtime state. This chapter is a comprehensive introduction to Colang, explaining all important concepts in a bottom up approach. -* [Introduction](introduction#reference_introduction) -* [Event Generation & Matching](event-generation-and-matching#event-generation-and-matching) -* [Working with Actions](working-with-actions#working-with-actions) -* [Defining Flows](defining-flows#defining-flows) -* [Working with Variables & Expressions](working-with-variables-and-expressions#working-with-variables-and-expressions) -* [Flow control](flow-control#flow-control) -* [Colang Standard Library (CSL)](the-standard-library#the-standard-library) -* [Make use of Large Language Models (LLM)](make-use-of-llms#make-use-of-llms) -* [More on Flows](more-on-flows#more-on-flows) -* [Python Actions](python-actions#python-actions) -* [Development and Debugging](development-and-debugging#development-and-debugging) +* [Introduction](/configure-guardrails/colang/colang-2/language-reference/introduction#reference_introduction) +* [Event Generation & Matching](/configure-guardrails/colang/colang-2/language-reference/event-generation-and-matching#event-generation-and-matching) +* [Working with Actions](/configure-guardrails/colang/colang-2/language-reference/working-with-actions#working-with-actions) +* [Defining Flows](/configure-guardrails/colang/colang-2/language-reference/defining-flows#defining-flows) +* [Working with Variables & Expressions](/configure-guardrails/colang/colang-2/language-reference/working-with-variables-and-expressions#working-with-variables-and-expressions) +* [Flow control](/configure-guardrails/colang/colang-2/language-reference/flow-control#flow-control) +* [Colang Standard Library (CSL)](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library) +* [Make use of Large Language Models (LLM)](/configure-guardrails/colang/colang-2/language-reference/make-use-of-llms#make-use-of-llms) +* [More on Flows](/configure-guardrails/colang/colang-2/language-reference/more-on-flows#more-on-flows) +* [Python Actions](/configure-guardrails/colang/colang-2/language-reference/python-actions#python-actions) +* [Development and Debugging](/configure-guardrails/colang/colang-2/language-reference/development-and-debugging#development-and-debugging) diff --git a/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx b/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx index 0a6d53dafa..94c21b46f5 100644 --- a/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx +++ b/docs/configure-rails/colang/colang-2/language-reference/make-use-of-llms.mdx @@ -10,7 +10,7 @@ description: "At its core, Colang does not require a Large Language Model (LLM) ## Introduction -At its core, Colang does not require a Large Language Model (LLM) as backend. However, many of the more advanced mechanisms in the [Colang Standard Library](the-standard-library#the-standard-library) (CSL) depend on one. +At its core, Colang does not require a Large Language Model (LLM) as backend. However, many of the more advanced mechanisms in the [Colang Standard Library](/configure-guardrails/colang/colang-2/language-reference/the-standard-library#the-standard-library) (CSL) depend on one. To enable the LLM backend, you first have to configure the LLM access in the `config.yml` by adding a `models` section like this: @@ -22,7 +22,7 @@ models: ``` Make sure to also define the required API access key. For example, for OpenAI you will have to set the `OPENAI_API_KEY` environment variable. -Every LLM prompt contains a default context that can be modified if needed to adapt to the use case. See this [example configuration](../../../../../tests/test_configs/multi_modal_demo_v2_x/demo.yml) to get started. This will heavily influence all the LLM invocations. +Every LLM prompt contains a default context that can be modified if needed to adapt to the use case. See this [example configuration](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/tests/test_configs/multi_modal_demo_v2_x/demo.yml) to get started. This will heavily influence all the LLM invocations. @@ -47,7 +47,7 @@ model: meta/llama3-70b-instruct model: meta/llama-3.1-8b-instruct model: meta/llama-3.1-70b-instruct ``` -To support other models, you would need to create a set of new [template prompts](../../../../../nemoguardrails/llm/prompts) that consider the specific capabilities and the API of the model and add them to your bot configuration. +To support other models, you would need to create a set of new [template prompts](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/nemoguardrails/llm/prompts) that consider the specific capabilities and the API of the model and add them to your bot configuration. ## Natural Language Description (NLD) @@ -88,7 +88,7 @@ flow main $question = await user said something ... ``` -See the example in [LLM Flows](../getting-started/llm-flows#colang_2_getting_started_llm_flows) for more details on how this works. +See the example in [LLM Flows](/configure-guardrails/colang/colang-2/getting-started/llm-flows#colang_2_getting_started_llm_flows) for more details on how this works. Note that there is no explicit control over the NLD response format and sometimes it will fail to generate the expected result. Usually you can improve it by providing more explicit instructions in the NLD, e.g. "Welcome the user with a short sentence that is wrapped in quotation marks like this: 'Hi there!'". Another way is to check the returned value by using for example the `is_str()` function to make sure that it is of the expected format. @@ -97,7 +97,7 @@ Note that there is no explicit control over the NLD response format and sometime ## User Intent Matching -In section [Defining Flows](defining-flows#action-like-and-intent-like-flows), we have already seen how we can define user intent flows. The limitation was that they did not generalize to variations of the given user intent examples. With the help of an LLM, we can overcome this issue and use its reasoning power by importing the `llm` standard library module and activating the flows `automating intent detection` and `generating user intent for unhandled user utterance` ([GitHub link](../../../../../nemoguardrails/colang/v2_x/library/llm.co)) to match unexpected user utterances to currently active user intent flows. +In section [Defining Flows](/configure-guardrails/colang/colang-2/language-reference/defining-flows#action-like-and-intent-like-flows), we have already seen how we can define user intent flows. The limitation was that they did not generalize to variations of the given user intent examples. With the help of an LLM, we can overcome this issue and use its reasoning power by importing the `llm` standard library module and activating the flows `automating intent detection` and `generating user intent for unhandled user utterance` ([GitHub link](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/colang/v2_x/library/llm.co)) to match unexpected user utterances to currently active user intent flows. **llm/user_intent_match_example/main.co** @@ -237,4 +237,4 @@ This loop will take care of matching user utterances to predefined user intents ## Guardrailing -Checkout the examples in the [Getting Started](../getting-started/index#getting_started) section or refer to the [NeMo Guardrails documentation](https://github.com/NVIDIA-NeMo/Guardrails) to learn more about how Colang can be used to guardrail LLM responses and user inputs. +Checkout the examples in the [Getting Started](/configure-guardrails/colang/colang-2/getting-started#getting_started) section or refer to the [NeMo Guardrails documentation](https://github.com/NVIDIA-NeMo/Guardrails) to learn more about how Colang can be used to guardrail LLM responses and user inputs. diff --git a/docs/configure-rails/colang/colang-2/language-reference/the-standard-library.mdx b/docs/configure-rails/colang/colang-2/language-reference/the-standard-library.mdx index 7361a28bf1..6e2bbbf626 100644 --- a/docs/configure-rails/colang/colang-2/language-reference/the-standard-library.mdx +++ b/docs/configure-rails/colang/colang-2/language-reference/the-standard-library.mdx @@ -8,41 +8,41 @@ description: "The Colang Standard Library (CSL) provides an abstraction from the -The Colang Standard Library (CSL) provides an abstraction from the underlying event and action layer and offers a semantic interface to design interaction patterns between the bot and the user. Currently, there are the following library files available under `nemoguardrails/colang/v2_x/library/` ([GitHub link](../../../../../nemoguardrails/colang/v2_x/library)): +The Colang Standard Library (CSL) provides an abstraction from the underlying event and action layer and offers a semantic interface to design interaction patterns between the bot and the user. Currently, there are the following library files available under `nemoguardrails/colang/v2_x/library/` ([GitHub link](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/nemoguardrails/colang/v2_x/library)): - + Core user, bot, wait, and notification flows. - + Wait, timer, and silence detection flows. - + LLM continuation and utility flows. - + Gesture, posture, and avatar-related flows. - + Standard guardrail flow definitions. - + User attention tracking and checks. diff --git a/docs/configure-rails/colang/index.mdx b/docs/configure-rails/colang/index.mdx index 59bd592f4d..45a2d8da3f 100644 --- a/docs/configure-rails/colang/index.mdx +++ b/docs/configure-rails/colang/index.mdx @@ -147,5 +147,5 @@ This event-driven interaction model is part of what makes Colang a powerful mode ## Getting Started -If you've used Colang 1.0 before, check out the [What's Changed with Colang 2.0](colang-2/whats-changed) page. -If not, you can get started with the Colang 2.0 [Hello World](colang-2/getting-started/hello-world) example. +If you've used Colang 1.0 before, check out the [What's Changed with Colang 2.0](/configure-guardrails/colang/colang-2/whats-changed) page. +If not, you can get started with the Colang 2.0 [Hello World](/configure-guardrails/colang/colang-2/getting-started/hello-world) example. diff --git a/docs/configure-rails/guardrail-catalog/community/ai-defense.mdx b/docs/configure-rails/guardrail-catalog/community/ai-defense.mdx index cad007bb12..eefe605df0 100644 --- a/docs/configure-rails/guardrail-catalog/community/ai-defense.mdx +++ b/docs/configure-rails/guardrail-catalog/community/ai-defense.mdx @@ -108,4 +108,4 @@ This allows you to choose between security (fail closed) and availability (fail ## Notes -For more information on Cisco AI Defense capabilities and configuration, please refer to the [Cisco AI Defense documentation](https://securitydocs.cisco.com/docs/scc/admin/108321.dita?utm_medium=github&utm_campaign=nemo-guardrails). +For more information on Cisco AI Defense capabilities and configuration, please refer to the [Cisco AI Defense documentation](https://developer.cisco.com/docs/ai-defense-inspection/). diff --git a/docs/configure-rails/guardrail-catalog/community/clavata.mdx b/docs/configure-rails/guardrail-catalog/community/clavata.mdx index f6cc1fd958..caded22467 100644 --- a/docs/configure-rails/guardrail-catalog/community/clavata.mdx +++ b/docs/configure-rails/guardrail-catalog/community/clavata.mdx @@ -105,7 +105,7 @@ flow input rails $input_text clavata check for ($input_text, Toxicity, ["Hate Speech","Harassment"]) ``` -> The same is true for `output` flows, of course. See [our example](../../../../examples/configs/clavata_v2/rails.co) for more. +> The same is true for `output` flows, of course. See [our example](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/examples/configs/clavata_v2/rails.co) for more. ### 2. Programmatic Usage @@ -141,4 +141,4 @@ If the Clavata API request fails, the system will raise a `ClavataPluginAPIError - You can configure different policies and label requirements for input and output flows - If no labels are specified for a policy, any label match will be considered a hit -For more information on Clavata and its capabilities, please refer to the [Clavata documentation](https://clavata.helpscoutdocs.com). +For more information on Clavata and its capabilities, please refer to the [Clavata documentation](https://docs.clavata.ai/). diff --git a/docs/configure-rails/guardrail-catalog/community/cleanlab.mdx b/docs/configure-rails/guardrail-catalog/community/cleanlab.mdx index 247c263302..b073230cc4 100644 --- a/docs/configure-rails/guardrail-catalog/community/cleanlab.mdx +++ b/docs/configure-rails/guardrail-catalog/community/cleanlab.mdx @@ -32,6 +32,6 @@ Install the Python client to use Cleanlab's trustworthiness score: pip install --upgrade cleanlab-tlm ``` -You can get an API key for free by [creating a Cleanlab account](https://tlm.cleanlab.ai/) or experiment with the trustworthiness scores in the [playground](https://chat.cleanlab.ai/chat). Feel free to [email Cleanlab](mailto:support@cleanlab.ai) with any questions. +You can get an API key for free by [creating a Cleanlab account](https://tlm.cleanlab.ai/) or experiment with the trustworthiness scores in the [playground](https://tlm.cleanlab.ai/). Feel free to [email Cleanlab](mailto:support@cleanlab.ai) with any questions. Lastly, set the `CLEANLAB_API_KEY` environment variable with the API key. diff --git a/docs/configure-rails/guardrail-catalog/community/fiddler.mdx b/docs/configure-rails/guardrail-catalog/community/fiddler.mdx index 242c08123d..b902b7d67e 100644 --- a/docs/configure-rails/guardrail-catalog/community/fiddler.mdx +++ b/docs/configure-rails/guardrail-catalog/community/fiddler.mdx @@ -3,15 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 title: "Fiddler Guardrails Integration" --- -Fiddler Guardrails utilizes [Fiddler Trust Models](https://docs.fiddler.ai/product-guide/llm-monitoring/llm-based-metrics#fiddler-fast-trust-metrics) in a specialized low-latency, high-throughput configuration. Guardrails can be used to guard Large Language Model (LLM) applications against user threats, such as prompt injection or harmful and inappropriate content, and LLM hallucinations. +Fiddler Guardrails utilizes [Fiddler Trust Models](https://docs.fiddler.ai/reference/glossary/centor-models) in a specialized low-latency, high-throughput configuration. Guardrails can be used to guard Large Language Model (LLM) applications against user threats, such as prompt injection or harmful and inappropriate content, and LLM hallucinations. -Currently, only Fiddler Trust Models ([Faithfulness](https://docs.fiddler.ai/product-guide/llm-monitoring/enrichments-private-preview#fast-faithfulness-private-preview) and [Safety](https://docs.fiddler.ai/product-guide/llm-monitoring/enrichments-private-preview#fast-safety-private-preview)) - Fiddler's in-house, purpose-built SLMs - are available for guardrail use. Future model releases and model updates/improvements will also be available for guardrail use. +Currently, only Fiddler Trust Models ([Faithfulness](https://docs.fiddler.ai/protection/guardrails) and [Safety](https://docs.fiddler.ai/protection/guardrails)) - Fiddler's in-house, purpose-built SLMs - are available for guardrail use. Future model releases and model updates/improvements will also be available for guardrail use. ## Setup 1. Ensure that you have access to a valid Fiddler environment. To obtain one, please [contact us](https://www.fiddler.ai/contact-sales). -2. Create a new [Fiddler environment key](https://docs.fiddler.ai/ui-guide/administration-ui/settings#credentials) and set the `FIDDLER_API_KEY` environment variable to this key to authenticate into the Fiddler service. +2. Create a new [Fiddler environment key](https://docs.fiddler.ai/protection/guardrails) and set the `FIDDLER_API_KEY` environment variable to this key to authenticate into the Fiddler service. Update your `config.yml` file to include the following settings: @@ -49,4 +49,4 @@ Fiddler Guardrails will not block inputs in the event of any API failure. ## Notes -For more information about Fiddler Guardrails, please visit the Fiddler Guardrails [documentation](https://docs.fiddler.ai/product-guide/llm-monitoring/guardrails). +For more information about Fiddler Guardrails, please visit the Fiddler Guardrails [documentation](https://docs.fiddler.ai/protection/guardrails). diff --git a/docs/configure-rails/guardrail-catalog/community/llama-guard.mdx b/docs/configure-rails/guardrail-catalog/community/llama-guard.mdx index 3fb1affacb..8ac8502bb0 100644 --- a/docs/configure-rails/guardrail-catalog/community/llama-guard.mdx +++ b/docs/configure-rails/guardrail-catalog/community/llama-guard.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Llama-Guard Integration" --- -NeMo Guardrails provides out-of-the-box support for content moderation using Meta's [Llama Guard](https://ai.meta.com/research/publications/llama-guard-llm-based-input-output-safeguard-for-human-ai-conversations/) model. +NeMo Guardrails provides out-of-the-box support for content moderation using Meta's [Llama Guard](https://arxiv.org/abs/2312.06674) model. In our testing, we observe significantly improved input and output content moderation performance compared to the [self-check method](/configure-guardrails/guardrail-catalog/self-check). Please see the [performance evaluation](/evaluation/evaluate-guardrails#llamaguard-based-moderation-rails-performance) for benchmark numbers. diff --git a/docs/configure-rails/guardrail-catalog/community/patronus-evaluate-api.mdx b/docs/configure-rails/guardrail-catalog/community/patronus-evaluate-api.mdx index 8fc44d6a6e..0795d96cb0 100644 --- a/docs/configure-rails/guardrail-catalog/community/patronus-evaluate-api.mdx +++ b/docs/configure-rails/guardrail-catalog/community/patronus-evaluate-api.mdx @@ -11,7 +11,7 @@ Patronus also has Managed configurations of the Judge evaluator, which you can u ## Setup 1. Sign up for an account on [app.patronus.ai](https://app.patronus.ai). -2. You can follow the Quick Start guide [here](https://docs.patronus.ai/docs/quickstart-guide) to get onboarded. +2. You can follow the Quick Start guide [here](https://docs.patronus.ai/docs/quickstart_eval) to get onboarded. 3. Create an API Key and save it somewhere safe. ## Usage diff --git a/docs/configure-rails/guardrail-catalog/content-safety.mdx b/docs/configure-rails/guardrail-catalog/content-safety.mdx index 6d4dd83531..27d90536bf 100644 --- a/docs/configure-rails/guardrail-catalog/content-safety.mdx +++ b/docs/configure-rails/guardrail-catalog/content-safety.mdx @@ -8,7 +8,7 @@ content: type: "Reference" --- -The content safety checks inside Guardrails act as a robust set of guardrails designed to ensure the integrity and safety of both input and output text. This feature allows users to utilize a variety of advanced content safety models such as Nvidia's [Nemotron Content Safety](https://docs.nvidia.com/nim/#nemoguard) model, Meta's [Llama Guard 3](https://www.llama.com/docs/model-cards-and-prompt-formats/llama-guard-3/), and Google's [ShieldGemma](https://ai.google.dev/gemma/docs/shieldgemma). +The content safety checks inside Guardrails act as a robust set of guardrails designed to ensure the integrity and safety of both input and output text. This feature allows users to utilize a variety of advanced content safety models such as Nvidia's [Nemotron Content Safety](https://docs.nvidia.com/nim/#nemoguard) model, Meta's [Llama Guard 3](https://github.com/meta-llama/PurpleLlama/blob/main/Llama-Guard3/8B/MODEL_CARD.md), and Google's [ShieldGemma](https://ai.google.dev/gemma/docs/shieldgemma). To use the content safety check, you should: @@ -115,7 +115,7 @@ To use the content safety check, you should: The above is an example prompt that you can use with the *content safety check input $model=shieldgemma*. The prompt has one input variable: `{{ user_input }}`, which includes user input that should be moderated. The completion must be "yes" if the response is not safe and "no" otherwise. Optionally, some models may return a set of policy violations. -The `content safety check input` and `content safety check output` rails executes the [`content_safety_check_input`](../../../nemoguardrails/library/content_safety/actions.py) and [`content_safety_check_output`](../../../nemoguardrails/library/content_safety/actions.py) actions respectively. +The `content safety check input` and `content safety check output` rails executes the [`content_safety_check_input`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/library/content_safety/actions.py) and [`content_safety_check_output`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/library/content_safety/actions.py) actions respectively. ## Reasoning Models as Content Safety Guards @@ -260,7 +260,7 @@ Language detection accuracy was benchmarked on two datasets: TODO: is this covered by the general content safety abstraction? -The NeMo Guardrails library provides out-of-the-box support for content moderation using Meta's [Llama Guard](https://ai.meta.com/research/publications/llama-guard-llm-based-input-output-safeguard-for-human-ai-conversations/) model. +The NeMo Guardrails library provides out-of-the-box support for content moderation using Meta's [Llama Guard](https://arxiv.org/abs/2312.06674) model. ### Example usage diff --git a/docs/configure-rails/guardrail-catalog/third-party.mdx b/docs/configure-rails/guardrail-catalog/third-party.mdx index 7df4867eb9..33b8a9f822 100644 --- a/docs/configure-rails/guardrail-catalog/third-party.mdx +++ b/docs/configure-rails/guardrail-catalog/third-party.mdx @@ -69,7 +69,7 @@ For more details, check out the [AutoAlign Integration](/configure-guardrails/gu ## Clavata -The NeMo Guardrails library supports using [Clavata AI](https://www.clavata.ai/blogs/partner-nvidia) as an input and output rail out-of-the-box (you need to have the CLAVATA_API_KEY environment variable set). +The NeMo Guardrails library supports using [Clavata AI](https://docs.clavata.ai/) as an input and output rail out-of-the-box (you need to have the CLAVATA_API_KEY environment variable set). ### Example usage @@ -145,9 +145,9 @@ For more details, check out the [GuardrailsAI Integration](/configure-guardrails ## Fiddler Guardrails for Safety and Hallucination Detection -The NeMo Guardrails library supports using [Fiddler Guardrails](https://docs.fiddler.ai/product-guide/llm-monitoring/guardrails) for safety and hallucination detection in input and output flows. +The NeMo Guardrails library supports using [Fiddler Guardrails](https://docs.fiddler.ai/protection/guardrails) for safety and hallucination detection in input and output flows. -In order to access Fiddler guardrails, you need access to a valid Fiddler environment, and a [Fiddler environment key](https://docs.fiddler.ai/ui-guide/administration-ui/settings#credentials). You'll need to set the `FIDDLER_API_KEY` environment variable to authenticate into the Fiddler service. +In order to access Fiddler guardrails, you need access to a valid Fiddler environment, and a [Fiddler environment key](https://docs.fiddler.ai/protection/guardrails). You'll need to set the `FIDDLER_API_KEY` environment variable to authenticate into the Fiddler service. ```yaml rails: diff --git a/docs/configure-rails/guardrail-catalog/topic-control.mdx b/docs/configure-rails/guardrail-catalog/topic-control.mdx index 02cfe3941e..6972d7d10c 100644 --- a/docs/configure-rails/guardrail-catalog/topic-control.mdx +++ b/docs/configure-rails/guardrail-catalog/topic-control.mdx @@ -48,7 +48,7 @@ To use the topic safety check, you should: - Do not answer enquiries unrelated to the company policies. ``` -The system prompt must end with the topic safety output restriction - `If any of the above conditions are violated, please respond with "off-topic". Otherwise, respond with "on-topic". You must respond with "on-topic" or "off-topic".` This condition is automatically added to the system prompt by the topic safety check input flow. If you want to customize the output restriction, you can do so by modifying the `TOPIC_SAFETY_OUTPUT_RESTRICTION` variable in the [`topic_safety_check_input`](../../../nemoguardrails/library/topic_safety/actions.py) action. +The system prompt must end with the topic safety output restriction - `If any of the above conditions are violated, please respond with "off-topic". Otherwise, respond with "on-topic". You must respond with "on-topic" or "off-topic".` This condition is automatically added to the system prompt by the topic safety check input flow. If you want to customize the output restriction, you can do so by modifying the `TOPIC_SAFETY_OUTPUT_RESTRICTION` variable in the [`topic_safety_check_input`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/library/topic_safety/actions.py) action. ## Customizing Topic Rules @@ -72,4 +72,4 @@ prompts: ## Implementation Details -The 'topic safety check input' flow uses the [`topic_safety_check_input`](../../../nemoguardrails/library/topic_safety/actions.py) action. The model returns a boolean value indicating whether the user input is on-topic or not. Please refer to the [topic safety example](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/examples/configs/topic_safety/README.md) for more details. +The 'topic safety check input' flow uses the [`topic_safety_check_input`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/library/topic_safety/actions.py) action. The model returns a boolean value indicating whether the user input is on-topic or not. Please refer to the [topic safety example](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/examples/configs/topic_safety/README.md) for more details. diff --git a/docs/evaluation/evaluate-guardrails.mdx b/docs/evaluation/evaluate-guardrails.mdx index 903f2bd318..c8267f932b 100644 --- a/docs/evaluation/evaluate-guardrails.mdx +++ b/docs/evaluation/evaluate-guardrails.mdx @@ -78,7 +78,7 @@ Important lessons to be learned from the evaluation results: - Each step in the three-step approach (user intent, next step/bot intent, bot message) used by Guardrails offers an improvement in performance. - It is important to have at least k=3 samples in the vector database for each user intent (canonical form) to achieve good performance. - Some models (e.g., gpt-3.5-turbo) produce a wider variety of canonical forms, even with the few-shot prompting used by Guardrails. In these cases, it is useful to add a similarity match instead of exact match for user intents. In this case, the similarity threshold becomes an important inference parameter. -- Initial results show that even small models, e.g. [dolly-v2-3b](https://huggingface.co/databricks/dolly-v2-3b), [vicuna-7b-v1.3](https://huggingface.co/lmsys/vicuna-7b-v1.3), [mpt-7b-instruct](https://huggingface.co/mosaicml/mpt-7b-instruct), [falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) have good performance for topical rails. +- Initial results show that even small models, e.g. dolly-v2-3b, [vicuna-7b-v1.3](https://huggingface.co/lmsys/vicuna-7b-v1.3), mpt-7b-instruct, [falcon-7b-instruct](https://huggingface.co/tiiuae/falcon-7b-instruct) have good performance for topical rails. - Using a single call for topical rails shows similar results to the default method (which uses up to 3 LLM calls for generating the final bot message) in most cases for `text-davinci-003` model. - Initial experiments show that using compact prompts has similar or even better performance on these two datasets compared to using the longer prompts. @@ -189,7 +189,7 @@ More details on how to set up the data in the right format and run the evaluatio Evaluation Date - Nov 23, 2023 (Mar 7 2024 for `gemini-1.0-pro`). -We evaluate the performance of the fact-checking rail on the [MSMARCO](https://huggingface.co/datasets/ms_marco) dataset using the Self-Check and the AlignScore approaches. To build the dataset, we randomly sample 100 (question, correct answer, evidence) triples, and then, for each triple, build a non-factual or incorrect answer to yield 100 (question, incorrect answer, evidence) triples. +We evaluate the performance of the fact-checking rail on the [MSMARCO](https://huggingface.co/datasets/microsoft/ms_marco) dataset using the Self-Check and the AlignScore approaches. To build the dataset, we randomly sample 100 (question, correct answer, evidence) triples, and then, for each triple, build a non-factual or incorrect answer to yield 100 (question, incorrect answer, evidence) triples. We breakdown the performance into positive entailment accuracy and negative entailment accuracy. Positive entailment accuracy is the accuracy of the model in correctly identifying answers that are grounded in the evidence passage. Negative entailment accuracy is the accuracy of the model in correctly identifying answers that are **not** supported in the evidence. Details on how to create synthetic negative examples can be found [here](https://github.com/NVIDIA-NeMo/Guardrails/tree/develop/nemoguardrails/evaluate/data/factchecking/README.md). @@ -343,7 +343,7 @@ To evaluate the hallucination rail on your own dataset, you can follow the creat #### Evaluation Results -To evaluate the hallucination rail, we manually curate a set of [questions](../../nemoguardrails/evaluate/data/hallucination/sample.txt) which mainly consists of questions with a false premise, i.e., questions that cannot have a correct answer. +To evaluate the hallucination rail, we manually curate a set of [questions](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/evaluate/data/hallucination/sample.txt) which mainly consists of questions with a false premise, i.e., questions that cannot have a correct answer. For example, the question "What is the capital of the moon?" has a false premise since the moon does not have a capital. Since the question is stated in a way that implies that the moon has a capital, the model might be tempted to make up a fact and answer the question. diff --git a/docs/evaluation/llm-vulnerability-scanning.mdx b/docs/evaluation/llm-vulnerability-scanning.mdx index 01c18e5107..fef0b9bc43 100644 --- a/docs/evaluation/llm-vulnerability-scanning.mdx +++ b/docs/evaluation/llm-vulnerability-scanning.mdx @@ -22,10 +22,10 @@ Think of Garak as an LLM alternative to network security scanners such as [nmap] The sample ABC guardrails configuration has been scanned using Garak against vulnerabilities, using four different configurations, offering increasing protection against LLM vulnerabilities: -1. **`bare_llm`**: no protection (full Garak results [here](./../_static/html/abc_bare_llm.report.html)). -2. **`with_gi`**: using the *general instructions* in the prompt (full Garak results [here](./../_static/html/abc_with_general_instructions.report.html)). -3. **`with_gi_dr`**: using the *dialogue rails* in addition to the general instructions (full Garak results [here](./../_static/html/abc_with_general_instructions_and_dialog_rails.report.html)). -4. **`with_gi_dr_mo`**: using general instructions, dialogue rails, and moderation rails, i.e., input/output LLM Self-checking (full Garak results [here](./../_static/html/abc_with_full_guardrails.report.html)). +1. **`bare_llm`**: no protection (full Garak results [here](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/docs/_static/html/abc_bare_llm.report.html)). +2. **`with_gi`**: using the *general instructions* in the prompt (full Garak results [here](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/docs/_static/html/abc_with_general_instructions.report.html)). +3. **`with_gi_dr`**: using the *dialogue rails* in addition to the general instructions (full Garak results [here](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/docs/_static/html/abc_with_general_instructions_and_dialog_rails.report.html)). +4. **`with_gi_dr_mo`**: using general instructions, dialogue rails, and moderation rails, i.e., input/output LLM Self-checking (full Garak results [here](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/docs/_static/html/abc_with_full_guardrails.report.html)). The table below summarizes what is included in each configuration: diff --git a/docs/getting-started/tutorials/jailbreak-detection-heuristics.mdx b/docs/getting-started/tutorials/jailbreak-detection-heuristics.mdx index 134a487b41..590dfe419d 100644 --- a/docs/getting-started/tutorials/jailbreak-detection-heuristics.mdx +++ b/docs/getting-started/tutorials/jailbreak-detection-heuristics.mdx @@ -63,7 +63,7 @@ rails: - self check input ``` -The `self check input` rail [prompts](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/docs/configure-rails/colang/colang-1/tutorials/6-topical-rails/config/prompts.yml) an LLM model to check if the input is safe for the bot to process. +The `self check input` rail [prompts](/configure-guardrails/guardrail-catalog/self-check#self-check-input) an LLM model to check if the input is safe for the bot to process. The `self check input` rail can be expensive to run for all input prompts, so you can use jailbreak detection heuristics as a low-latency and low-cost alternative to filter out malicious prompts. ## Jailbreak Detection Heuristics diff --git a/docs/integration/langchain/agent-middleware.mdx b/docs/integration/langchain/agent-middleware.mdx index 21cf136c12..e482f71aeb 100644 --- a/docs/integration/langchain/agent-middleware.mdx +++ b/docs/integration/langchain/agent-middleware.mdx @@ -305,7 +305,7 @@ guardrails = GuardrailsMiddleware( ) ``` -For more details, see [Security Considerations](https://docs.nvidia.com/nemo/guardrails/latest/integration/tools-integration.html#security-considerations) in the tools integration guide. +For more details, see [Security Considerations](/integration-with-third-party-libraries/tools-integration#security-considerations) in the tools integration guide. ### MODIFIED Status Replaces Message Content diff --git a/docs/resources/research.mdx b/docs/resources/research.mdx index fd66af364c..b189d674f9 100644 --- a/docs/resources/research.mdx +++ b/docs/resources/research.mdx @@ -17,7 +17,7 @@ For each category we present a list of relevant surveys, existing research paper Relevant surveys on hallucination detection and checking factuality for large language models. -- Ji, Z., Lee, N., Frieske, R., Yu, T., Su, D., Xu, Y., ... & Fung, P. (2023). Survey of hallucination in natural language generation. ACM Computing Surveys, 55(12), 1-38. [paper](https://dl.acm.org/doi/pdf/10.1145/3571730) +- Ji, Z., Lee, N., Frieske, R., Yu, T., Su, D., Xu, Y., ... & Fung, P. (2023). Survey of hallucination in natural language generation. ACM Computing Surveys, 55(12), 1-38. [paper](https://arxiv.org/abs/2202.03629) - Zhang, Y., Li, Y., Cui, L., Cai, D., Liu, L., Fu, T., ... & Shi, S. (2023). Siren's song in the AI ocean: a survey on hallucination in large language models. arXiv preprint arXiv:2309.01219. [paper](https://arxiv.org/pdf/2309.01219.pdf) - Huang, L., Yu, W., Ma, W., Zhong, W., Feng, Z., Wang, H., ... & Liu, T. (2023). A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions. arXiv preprint arXiv:2311.05232. [paper](https://arxiv.org/pdf/2311.05232.pdf) - Wang, C., Liu, X., Yue, Y., Tang, X., Zhang, T., Jiayang, C., ... & Zhang, Y. (2023). Survey on factuality in large language models: Knowledge, retrieval and domain-specificity. arXiv preprint arXiv:2310.07521. [paper](https://arxiv.org/pdf/2310.07521.pdf) | [repo](https://github.com/wangcunxiang/LLM-Factuality-Survey) diff --git a/docs/telemetry.mdx b/docs/telemetry.mdx index 651796662d..a268147feb 100644 --- a/docs/telemetry.mdx +++ b/docs/telemetry.mdx @@ -158,4 +158,4 @@ This behavior keeps adoption metrics focused on real deployments, not synthetic ## Schema and Source Code -The Python source for the event lives in [`nemoguardrails/telemetry.py`](../nemoguardrails/telemetry.py). The wire contract is validated against a vendored snapshot of the shared NVIDIA telemetry schema at [`schemas/anonymous_events.snapshot.json`](../schemas/anonymous_events.snapshot.json). The conformance test in [`tests/telemetry/test_usage_reporting.py`](../tests/telemetry/test_usage_reporting.py) validates emitted payloads against that snapshot with `jsonschema`. +The Python source for the event lives in [`nemoguardrails/telemetry.py`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/nemoguardrails/telemetry.py). The wire contract is validated against a vendored snapshot of the shared NVIDIA telemetry schema at [`schemas/anonymous_events.snapshot.json`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/schemas/anonymous_events.snapshot.json). The conformance test in [`tests/telemetry/test_usage_reporting.py`](https://github.com/NVIDIA-NeMo/Guardrails/blob/develop/tests/telemetry/test_usage_reporting.py) validates emitted payloads against that snapshot with `jsonschema`. diff --git a/fern/docs.yml b/fern/docs.yml index 204bbe0d72..bd951d0e6d 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -57,6 +57,79 @@ redirects: destination: "/nemo/guardrails/latest" - source: "/nemo/guardrails/latest/index" destination: "/nemo/guardrails/latest" + # Previous About section slug. + - source: "/nemo/guardrails/latest/about" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about/overview" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about/overview" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about/overview.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about/overview.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about/overview/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/about/overview/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/overview" + - source: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/latest/about/how-it-works.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/about/how-it-works.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/latest/about/how-it-works/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/about/how-it-works/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" + - source: "/nemo/guardrails/latest/about/rail-types" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/about/rail-types" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/latest/about/rail-types.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/about/rail-types.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/latest/about/rail-types/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/about/rail-types/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/rail-types" + - source: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/latest/about/supported-llms.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/about/supported-llms.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/latest/about/supported-llms/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/about/supported-llms/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" + - source: "/nemo/guardrails/latest/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" + - source: "/nemo/guardrails/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" + - source: "/nemo/guardrails/latest/about/release-notes.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" + - source: "/nemo/guardrails/about/release-notes.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" + - source: "/nemo/guardrails/latest/about/release-notes/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" + - source: "/nemo/guardrails/about/release-notes/index.html" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" # Legacy Sphinx redirects from docs/conf.py. - source: "/nemo/guardrails/latest/introduction" destination: "/nemo/guardrails/latest" @@ -99,13 +172,13 @@ redirects: - source: "/nemo/guardrails/user-guides/advanced/nemoguard-contentsafety-deployment.html" destination: "/nemo/guardrails/latest/getting-started/tutorials/nemotron-safety-guard-deployment" - source: "/nemo/guardrails/latest/architecture" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/architecture" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/latest/architecture.html" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/architecture.html" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/latest/architecture/readme" destination: "/nemo/guardrails/latest/reference/colang-architecture-guide" - source: "/nemo/guardrails/architecture/readme" @@ -131,13 +204,13 @@ redirects: - source: "/nemo/guardrails/glossary.html" destination: "/nemo/guardrails/latest" - source: "/nemo/guardrails/latest/release-notes" - destination: "/nemo/guardrails/latest/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" - source: "/nemo/guardrails/release-notes" - destination: "/nemo/guardrails/latest/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" - source: "/nemo/guardrails/latest/release-notes.html" - destination: "/nemo/guardrails/latest/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" - source: "/nemo/guardrails/release-notes.html" - destination: "/nemo/guardrails/latest/about/release-notes" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/release-notes" - source: "/nemo/guardrails/latest/security/guidelines" destination: "/nemo/guardrails/latest/resources/security/guidelines" - source: "/nemo/guardrails/security/guidelines" @@ -891,13 +964,13 @@ redirects: - source: "/nemo/guardrails/user-guides/advanced/using-docker.html" destination: "/nemo/guardrails/latest/deployment/using-docker" - source: "/nemo/guardrails/latest/user-guides/advanced/vertexai-setup" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/advanced/vertexai-setup" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/advanced/vertexai-setup.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/advanced/vertexai-setup.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/cli" destination: "/nemo/guardrails/latest/reference/cli" - source: "/nemo/guardrails/user-guides/cli" @@ -923,61 +996,61 @@ redirects: - source: "/nemo/guardrails/user-guides/guardrails-library.html" destination: "/nemo/guardrails/latest/configure-rails/guardrail-catalog" - source: "/nemo/guardrails/latest/user-guides/guardrails-process" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/user-guides/guardrails-process" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/latest/user-guides/guardrails-process.html" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/user-guides/guardrails-process.html" - destination: "/nemo/guardrails/latest/about/how-it-works" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/how-it-works" - source: "/nemo/guardrails/latest/user-guides/llm-support" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm-support" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm-support.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm-support.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/nvidia-ai-endpoints" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/nvidia-ai-endpoints" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/nvidia-ai-endpoints.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/nvidia-ai-endpoints.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/nvidia-ai-endpoints/readme" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/nvidia-ai-endpoints/readme" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/nvidia-ai-endpoints/readme.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/nvidia-ai-endpoints/readme.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/vertexai" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/vertexai" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/vertexai.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/vertexai.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/vertexai/readme" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/vertexai/readme" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/llm/vertexai/readme.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/user-guides/llm/vertexai/readme.html" - destination: "/nemo/guardrails/latest/about/supported-llms" + destination: "/nemo/guardrails/latest/about-nemo-guardrails-library/supported-llms" - source: "/nemo/guardrails/latest/user-guides/migration-guide" destination: "/nemo/guardrails/latest/configure-rails/colang/colang-2/migration-guide" - source: "/nemo/guardrails/user-guides/migration-guide" @@ -1010,6 +1083,51 @@ redirects: destination: "/nemo/guardrails/latest/getting-started/tutorials/multimodal" - source: "/nemo/guardrails/user-guides/multimodal.html" destination: "/nemo/guardrails/latest/getting-started/tutorials/multimodal" + # Colang 2.0 language-reference pages previously rendered one level higher. + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/introduction" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/introduction" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/introduction" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/introduction" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/working-with-actions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/working-with-actions" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/working-with-actions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/working-with-actions" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/event-generation-and-matching" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/event-generation-and-matching" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/event-generation-and-matching" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/event-generation-and-matching" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/defining-flows" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/defining-flows" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/defining-flows" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/defining-flows" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/working-with-variables-and-expressions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/working-with-variables-and-expressions" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/working-with-variables-and-expressions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/working-with-variables-and-expressions" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/the-standard-library" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/the-standard-library" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/the-standard-library" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/the-standard-library" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/flow-control" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/flow-control" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/flow-control" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/flow-control" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/make-use-of-llms" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/make-use-of-llms" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/make-use-of-llms" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/make-use-of-llms" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/development-and-debugging" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/development-and-debugging" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/development-and-debugging" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/development-and-debugging" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/python-actions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/python-actions" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/python-actions" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/python-actions" + - source: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/more-on-flows" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/more-on-flows" + - source: "/nemo/guardrails/configure-guardrails/colang/colang-2/more-on-flows" + destination: "/nemo/guardrails/latest/configure-guardrails/colang/colang-2/language-reference/more-on-flows" - source: "/nemo/guardrails/latest/user-guides/python-api" destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" - source: "/nemo/guardrails/user-guides/python-api" @@ -1050,6 +1168,52 @@ redirects: destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/streaming" - source: "/nemo/guardrails/api/nemoguardrails.streaming.html" destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/streaming" + # Generated SDK pages can emit links to private helper modules that Fern does + # not publish as standalone pages. Route them to the nearest public SDK page. + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/_compat" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/_compat" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/embeddings" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/embeddings" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/__main__" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/__main__" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/deal__status" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/comd_parser" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/deal__status" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/comd_parser" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/user=CURRENT" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/comd_parser" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/user=CURRENT" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/colang/v1_0/lang/comd_parser" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/guardrails/_http" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/guardrails" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/guardrails/_http" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/guardrails" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/helpers" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/llm/helpers" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients/_errors" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/llm/clients/_errors" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients/_sse" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/llm/clients/_sse" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/clients" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers/huggingface" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/llm/providers/huggingface" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers" + - source: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers/trtllm" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers" + - source: "/nemo/guardrails/guardrails-python-sdk/nemoguardrails/llm/providers/trtllm" + destination: "/nemo/guardrails/latest/guardrails-python-sdk/nemoguardrails/llm/providers" - source: "/nemo/guardrails/:path*/index.html" destination: "/nemo/guardrails/:path*" - source: "/nemo/guardrails/:path*.html" diff --git a/nemoguardrails/colang/v1_0/lang/comd_parser.py b/nemoguardrails/colang/v1_0/lang/comd_parser.py index cb086e4ace..a611894adc 100644 --- a/nemoguardrails/colang/v1_0/lang/comd_parser.py +++ b/nemoguardrails/colang/v1_0/lang/comd_parser.py @@ -26,15 +26,11 @@ def parse_pattern(pattern): """ Parses a pattern from the Markdown-friendly format to an internal format. - E.g. parse_pattern("show [me](user=CURRENT) the deals I've [won](deal__status)") = - (show me the deals I've {deal__status}, {'user': 'CURRENT'}). - - For patters with "assignment patterns" like "show [me](user=CURRENT) the deals" we - transform it into: - "show {user=CURRENT} the deals" with the mapping: - { - "user=CURRENT": "me" - } + For example, a pattern with an implicit user assignment and a captured deal + status returns `("show me the deals I've {deal__status}", {"user": "CURRENT"})`. + + Assignment patterns are transformed into capture placeholders and record + an implicit mapping such as `{"user=CURRENT": "me"}`. :param pattern: The pattern in Markdown-friendly format. :return: A tuple (pattern, params) where pattern is a pattern containing only diff --git a/nemoguardrails/tracing/constants.py b/nemoguardrails/tracing/constants.py index 2cc954638a..bc44f66c91 100644 --- a/nemoguardrails/tracing/constants.py +++ b/nemoguardrails/tracing/constants.py @@ -308,7 +308,7 @@ class EventNames: """Standard event names for OpenTelemetry GenAI semantic conventions. Based on official spec at: - https://github.com/open-telemetry/semantic-conventions/blob/main/model/gen-ai/events.yaml + https://opentelemetry.io/docs/concepts/semantic-conventions/ """ GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" diff --git a/nemoguardrails/utils.py b/nemoguardrails/utils.py index fee24614c5..9047b06e21 100644 --- a/nemoguardrails/utils.py +++ b/nemoguardrails/utils.py @@ -118,8 +118,9 @@ def _has_property(e: Dict[str, Any], p: Property) -> bool: ), Validator( "***UtteranceUserActionTranscriptUpdated events need to provide 'interim_transcript' of type 'str'", - lambda e: e["type"] != "UtteranceUserActionTranscriptUpdated" - or _has_property(e, Property("interim_transcript", str)), + lambda e: ( + e["type"] != "UtteranceUserActionTranscriptUpdated" or _has_property(e, Property("interim_transcript", str)) + ), ), Validator( "***UtteranceUserActionFinished events need to provide 'final_transcript' of type 'str'", diff --git a/scripts/check-docs-links.sh b/scripts/check-docs-links.sh deleted file mode 100755 index 87f7e4d48c..0000000000 --- a/scripts/check-docs-links.sh +++ /dev/null @@ -1,719 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Markdown/MDX link checker for Guardrails docs. -# -# Usage: -# scripts/check-docs-links.sh -# scripts/check-docs-links.sh --local-only -# scripts/check-docs-links.sh path/to/page.md path/to/page.mdx -# -# Environment: -# CHECK_DOC_LINKS_REMOTE If 0, skip http(s) probes. -# CHECK_DOC_LINKS_VERBOSE If 1, log each URL while curling. -# CHECK_DOC_LINKS_IGNORE_EXTRA Comma-separated extra http(s) URLs to skip. -# CHECK_DOC_LINKS_IGNORE_URL_REGEX Skip remote probes when the full URL matches this ERE. -# CHECK_DOCS_FERN_NAV_YML Override docs/index.yml for tests. -# NODE Node binary for Fern route parsing. -# CURL curl binary for remote probes. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || true)" -if [[ -z "${REPO_ROOT:-}" ]]; then - REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -fi - -CURL="${CURL:-curl}" -NODE="${NODE:-node}" -CHECK_DOC_LINKS_REMOTE="${CHECK_DOC_LINKS_REMOTE:-1}" -VERBOSE="${CHECK_DOC_LINKS_VERBOSE:-0}" -EXTRA_FILES=() - -usage() { - cat <<'EOF' -Markdown/MDX link checker for Guardrails docs. - -Usage: scripts/check-docs-links.sh [options] [extra.md/.mdx ...] - -Options: - --local-only Do not curl http(s) URLs (same as CHECK_DOC_LINKS_REMOTE=0). - --verbose Log each URL while curling. - -h, --help Show this help. - -Environment: CHECK_DOC_LINKS_REMOTE, CHECK_DOC_LINKS_VERBOSE, - CHECK_DOC_LINKS_IGNORE_EXTRA, CHECK_DOC_LINKS_IGNORE_URL_REGEX, - CHECK_DOCS_FERN_NAV_YML, NODE, CURL. -EOF -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --local-only) - CHECK_DOC_LINKS_REMOTE=0 - shift - ;; - --verbose) - VERBOSE=1 - shift - ;; - -h | --help) - usage - exit 0 - ;; - --) - shift - EXTRA_FILES+=("$@") - break - ;; - -*) - echo "check-docs-links: unknown option: $1" >&2 - usage >&2 - exit 2 - ;; - *) - EXTRA_FILES+=("$1") - shift - ;; - esac -done - -log() { - printf '%s\n' "check-docs-links: $*" -} - -collect_default_docs() { - local f - for f in \ - "$REPO_ROOT/README.md" \ - "$REPO_ROOT/CONTRIBUTING.md" \ - "$REPO_ROOT/SECURITY.md" \ - "$REPO_ROOT/CHANGELOG.md"; do - [[ -f "$f" ]] && printf '%s\n' "$f" - done - if [[ -d "$REPO_ROOT/docs" ]]; then - find "$REPO_ROOT/docs" \ - -path "$REPO_ROOT/docs/_static/python-sdk-reference" -prune \ - -o -type f \( -name '*.md' -o -name '*.mdx' \) -print | LC_ALL=C sort - fi -} - -extract_targets() { - LC_ALL=C perl -CS -ne ' - if ($in_fence) { - if (/^\s*(`{3,}|~{3,})(.*)$/) { - my $fence = $1; - my $rest = $2; - my $char = substr($fence, 0, 1); - my $length = length($fence); - if ($char eq $fch && $length >= $flen && $rest =~ /^\s*$/) { - ($in_fence, $fch, $flen) = (0, "", 0); - } - } - next; - } - - my $line = $.; - my $text = $_; - my $visible = ""; - - while (length $text) { - if ($in_comment) { - if ($text =~ s/^(.*?)-->//s) { - $in_comment = 0; - next; - } - $text = ""; - next; - } - - if ($text =~ s/^(.*?)/) { - die "malformed HTML comment\n"; - } - - $visible .= $text; - last; - } - - if ($visible =~ /^\s*(`{3,}|~{3,})(.*)$/) { - my $fence = $1; - my $char = substr($fence, 0, 1); - my $length = length($fence); - ($in_fence, $fch, $flen) = (1, $char, $length); - next; - } - - my $scan = $visible; - $scan =~ s/`[^`]*`//g; - while ($scan =~ /\!?\[[^\]]*\]\(([^)\s]+)(?:\s+["'"'"'][^)"'"'"']*["'"'"'])?\)/g) { print $line . "\t" . $1 . "\n"; } - while ($scan =~ /<(https?:[^>\s]+)>/g) { print $line . "\t" . $1 . "\n"; } - while ($scan =~ /\bhref=(["'"'"'])([^"'"'"'\s]+)\1/g) { print $line . "\t" . $2 . "\n"; } - END { - die "malformed HTML comment\n" if $in_comment; - } - ' -- "$1" -} - -FERN_ROUTE_INDEX_LOADED=0 -FERN_ROUTE_INDEX="" - -load_fern_route_index() { - [[ "$FERN_ROUTE_INDEX_LOADED" -eq 1 ]] && return 0 - FERN_ROUTE_INDEX_LOADED=1 - - local nav_yml="${CHECK_DOCS_FERN_NAV_YML:-$REPO_ROOT/docs/index.yml}" - [[ -f "$nav_yml" ]] || return 0 - if ! command -v "$NODE" >/dev/null 2>&1; then - return 0 - fi - - local _fern_route_index_err - _fern_route_index_err="$(mktemp)" - if ! FERN_ROUTE_INDEX="$( - "$NODE" - "$REPO_ROOT" "$nav_yml" <<'NODE' 2>"$_fern_route_index_err" -const fs = require("node:fs"); -const path = require("node:path"); - -const repoRoot = process.argv[2]; -const navPath = process.argv[3]; -const docsRoot = path.join(repoRoot, "docs"); -const rows = []; -const routes = new Set(); - -function clean(value) { - let out = value.trim(); - const hash = out.indexOf(" #"); - if (hash >= 0) out = out.slice(0, hash).trim(); - if ((out.startsWith('"') && out.endsWith('"')) || (out.startsWith("'") && out.endsWith("'"))) { - out = out.slice(1, -1); - } - return out; -} - -function normalizeRoute(input) { - let out = input.replace(/\\/g, "/").replace(/^\/+/, ""); - out = out.replace(/\.(?:md|mdx)$/i, ""); - out = out.replace(/\/index$/i, ""); - return out; -} - -function emit(source, route) { - route = normalizeRoute(route); - if (!route) return; - rows.push(`${source}\t${route}`); - routes.add(route); -} - -function emitDocsIndexRoutes() { - const lines = fs.readFileSync(navPath, "utf8").split(/\r?\n/); - let stack = []; - let current = null; - for (const line of lines) { - const itemMatch = line.match(/^(\s*)-\s+(page|section):/); - if (itemMatch) { - const indent = itemMatch[1].length; - while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop(); - current = { - indent, - type: itemMatch[2], - parent: stack.map((part) => part.slug), - path: "", - slug: "", - emitted: false, - pushed: false, - }; - continue; - } - - const propMatch = line.match(/^(\s*)(path|slug):\s*(.+?)\s*$/); - if (!propMatch || !current) continue; - const indent = propMatch[1].length; - if (indent !== current.indent + 2) continue; - - const key = propMatch[2]; - const value = clean(propMatch[3]); - if (key === "path") current.path = value; - if (key === "slug") current.slug = value; - if (!current.emitted && current.path && current.slug) { - emit(current.path, [...current.parent, current.slug].join("/")); - current.emitted = true; - } - if (!current.pushed && current.type === "section" && current.slug) { - stack.push({ indent: current.indent, slug: current.slug }); - current.pushed = true; - } - } -} - -function walk(directory) { - if (!fs.existsSync(directory)) return; - for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { - const fullPath = path.join(directory, entry.name); - if (entry.isDirectory()) { - walk(fullPath); - } else if (entry.isFile() && entry.name.endsWith(".mdx")) { - emitFrontmatterSlug(fullPath); - } - } -} - -function emitFrontmatterSlug(fullPath) { - const rel = path.relative(docsRoot, fullPath).replace(/\\/g, "/"); - const content = fs.readFileSync(fullPath, "utf8"); - const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); - if (!match) return; - const slugMatch = match[1].match(/^slug:\s*(.+?)\s*$/m); - if (!slugMatch) return; - emit(rel, clean(slugMatch[1])); -} - -function emitGeneratedNavigationRoutes() { - const navFiles = [ - path.join(docsRoot, "_static", "python-sdk-reference", "_navigation.yml"), - ]; - for (const generatedNav of navFiles) { - if (!fs.existsSync(generatedNav)) continue; - const lines = fs.readFileSync(generatedNav, "utf8").split(/\r?\n/); - let currentSlug = ""; - for (const line of lines) { - const slugMatch = line.match(/^\s*slug:\s*(.+?)\s*$/); - if (slugMatch) { - currentSlug = clean(slugMatch[1]); - routes.add(normalizeRoute(currentSlug)); - continue; - } - const pageMatch = line.match(/^\s*pageId:\s*(.+?)\s*$/); - if (pageMatch && currentSlug) { - emit(`_static/python-sdk-reference/${clean(pageMatch[1])}`, currentSlug); - } - } - } -} - -emitDocsIndexRoutes(); -emitGeneratedNavigationRoutes(); -walk(path.join(docsRoot, "_static", "python-sdk-reference", "guardrails-python-sdk")); - -for (const route of routes) { - rows.push(`\t${route}`); -} - -if (rows.length === 0) { - throw new Error(`no Fern routes found in ${navPath}`); -} -process.stdout.write(rows.join("\n")); -NODE - )"; then - echo "check-docs-links: failed to parse Fern navigation ${nav_yml#"$REPO_ROOT"/}: $(tr '\n' ' ' <"$_fern_route_index_err" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//')" >&2 - rm -f "$_fern_route_index_err" - return 1 - fi - rm -f "$_fern_route_index_err" -} - -normalize_fern_route_path() { - local input="$1" part - input="${input#/}" - case "$input" in - nemo/guardrails/latest/*) input="${input#nemo/guardrails/latest/}" ;; - nemo/guardrails/*) input="${input#nemo/guardrails/}" ;; - latest/*) input="${input#latest/}" ;; - esac - input="${input%.mdx}" - input="${input%.md}" - input="${input%/index}" - - local -a parts=() out=() - local IFS='/' - read -r -a parts <<<"$input" - unset IFS - for part in "${parts[@]}"; do - case "$part" in - "" | .) ;; - ..) - if [[ "${#out[@]}" -eq 0 ]]; then - return 1 - fi - unset 'out[${#out[@]}-1]' - ;; - *) out+=("$part") ;; - esac - done - - local joined - joined="$( - IFS=/ - printf '%s' "${out[*]}" - )" - printf '%s' "$joined" -} - -fern_route_exists() { - local route="$1" - if ! load_fern_route_index; then - return 3 - fi - [[ -n "$FERN_ROUTE_INDEX" ]] || return 1 - - route="$(normalize_fern_route_path "$route")" || return 1 - while IFS=$'\t' read -r _source indexed_route || [[ -n "${indexed_route:-}" ]]; do - [[ "$indexed_route" == "$route" ]] && return 0 - done <<<"$FERN_ROUTE_INDEX" - return 1 -} - -fern_relative_ref_exists() { - local md_path="$1" stripped="$2" - local abs_md="$md_path" source_rel current route base - [[ "$abs_md" == /* ]] || abs_md="$REPO_ROOT/$abs_md" - case "$abs_md" in - "$REPO_ROOT/docs/"*) source_rel="${abs_md#"$REPO_ROOT/docs/"}" ;; - *) return 1 ;; - esac - - if ! load_fern_route_index; then - return 3 - fi - [[ -n "$FERN_ROUTE_INDEX" ]] || return 1 - - while IFS=$'\t' read -r _source current || [[ -n "${current:-}" ]]; do - [[ "$_source" == "$source_rel" ]] || continue - base="${current%/*}" - [[ "$base" == "$current" ]] && base="" - route="${base:+$base/}$stripped" - local _fern_rc - if fern_route_exists "$route"; then - _fern_rc=0 - else - _fern_rc=$? - fi - if [[ "$_fern_rc" -eq 0 ]]; then - return 0 - elif [[ "$_fern_rc" -eq 3 ]]; then - return 3 - fi - done <<<"$FERN_ROUTE_INDEX" - return 1 -} - -source_ref_exists() { - local base_dir="$1" stripped="$2" candidate - local -a candidates=("$stripped") - if [[ "$stripped" == */ ]]; then - candidates+=("${stripped}index.mdx" "${stripped}index.md") - else - candidates+=("$stripped.mdx" "$stripped.md" "$stripped/index.mdx" "$stripped/index.md") - fi - - for candidate in "${candidates[@]}"; do - if (cd "$base_dir" && [[ -e "$candidate" ]]); then - return 0 - fi - done - return 1 -} - -site_source_ref_exists() { - local stripped="$1" - local site_path="${stripped#/}" - local -a site_paths=("$site_path") - case "$site_path" in - nemo/guardrails/latest/*) site_paths+=("${site_path#nemo/guardrails/latest/}") ;; - nemo/guardrails/*) site_paths+=("${site_path#nemo/guardrails/}") ;; - latest/*) site_paths+=("${site_path#latest/}") ;; - esac - - local route_path - for route_path in "${site_paths[@]}"; do - if source_ref_exists "$REPO_ROOT/docs" "$route_path"; then - return 0 - fi - done - return 1 -} - -has_markdown_extension() { - case "$1" in - *.md | *.mdx) return 0 ;; - *) return 1 ;; - esac -} - -check_local_ref() { - local md_path="$1" line_no="$2" target="$3" - local stripped - - stripped="${target%%\#*}" - stripped="${stripped%%\?*}" - - [[ -z "$stripped" ]] && return 0 - [[ "$stripped" == api:* ]] && return 0 - [[ "$stripped" == mailto:* ]] && return 0 - [[ "$stripped" == tel:* ]] && return 0 - [[ "$stripped" == javascript:* ]] && return 0 - - if [[ "$stripped" == http://* || "$stripped" == https://* ]]; then - return 2 - fi - if [[ "$stripped" == *://* ]]; then - return 0 - fi - - if [[ "$stripped" == /* ]]; then - if [[ "$stripped" == /guardrails-python-sdk/* ]]; then - return 0 - fi - local _fern_rc - if fern_route_exists "$stripped"; then - _fern_rc=0 - else - _fern_rc=$? - fi - if [[ "$_fern_rc" -eq 0 ]] && has_markdown_extension "$stripped"; then - echo "check-docs-links: route-style link should omit .md/.mdx extension in $md_path:$line_no -> $target" >&2 - return 1 - fi - if [[ "$_fern_rc" -eq 0 ]]; then - return 0 - elif [[ "$_fern_rc" -eq 3 ]]; then - return 1 - fi - echo "check-docs-links: broken site route in $md_path:$line_no -> $target" >&2 - return 1 - fi - - local _fern_relative_rc - if fern_relative_ref_exists "$md_path" "$stripped"; then - _fern_relative_rc=0 - else - _fern_relative_rc=$? - fi - if [[ "$_fern_relative_rc" -eq 0 ]] && has_markdown_extension "$stripped"; then - echo "check-docs-links: route-style link should omit .md/.mdx extension in $md_path:$line_no -> $target" >&2 - return 1 - fi - if [[ "$_fern_relative_rc" -eq 0 ]]; then - return 0 - elif [[ "$_fern_relative_rc" -eq 3 ]]; then - return 1 - fi - if source_ref_exists "$(dirname "$md_path")" "$stripped"; then - return 0 - fi - echo "check-docs-links: broken local link in $md_path:$line_no -> $target" >&2 - return 1 -} - -check_remote_url() { - local url="$1" - if ! command -v "$CURL" >/dev/null 2>&1; then - echo "check-docs-links: curl not found; cannot verify $url" >&2 - return 1 - fi - if ! "$CURL" -fsS -L -o /dev/null \ - --connect-timeout 12 --max-time 35 \ - -A 'Guardrails-doc-link-check/1.0 (+https://github.com/NVIDIA-NeMo/Guardrails)' \ - "$url" 2>/dev/null; then - echo "check-docs-links: unreachable URL: $url" >&2 - return 1 - fi - return 0 -} - -normalize_url_for_ignore_match() { - local u="$1" - u="${u%%\#*}" - u="${u%/}" - printf '%s' "$u" -} - -check_docs_default_ignored_urls() { - printf '%s\n' \ - 'https://github.com/NVIDIA-NeMo/Guardrails/commits/develop' \ - 'https://github.com/NVIDIA-NeMo/Guardrails/pulls?q=is%3Apr+is%3Amerged' \ - 'https://github.com/NVIDIA-NeMo/Guardrails/pulls?q=is:pr+is:merged' -} - -url_should_skip_remote_probe() { - local url="$1" - local nu ign _re - nu="$(normalize_url_for_ignore_match "$url")" - - while IFS= read -r ign || [[ -n "${ign:-}" ]]; do - [[ -z "${ign:-}" ]] && continue - [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 - done < <(check_docs_default_ignored_urls) - - if [[ -n "${CHECK_DOC_LINKS_IGNORE_EXTRA:-}" ]]; then - local -a _extra_parts=() - local IFS=',' - read -ra _extra_parts <<<"${CHECK_DOC_LINKS_IGNORE_EXTRA}" - unset IFS - for ign in "${_extra_parts[@]}"; do - ign="${ign#"${ign%%[![:space:]]*}"}" - ign="${ign%"${ign##*[![:space:]]}"}" - [[ -z "$ign" ]] && continue - [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 - done - fi - - if [[ -n "${CHECK_DOC_LINKS_IGNORE_URL_REGEX:-}" ]]; then - _re="${CHECK_DOC_LINKS_IGNORE_URL_REGEX}" - [[ "$url" =~ $_re ]] && return 0 - fi - - return 1 -} - -run_links_check() { - local -a DOC_FILES - if [[ ${#EXTRA_FILES[@]} -gt 0 ]]; then - DOC_FILES=("${EXTRA_FILES[@]}") - else - DOC_FILES=() - while IFS= read -r _docf || [[ -n "${_docf:-}" ]]; do - [[ -z "${_docf:-}" ]] && continue - DOC_FILES+=("$_docf") - done < <(collect_default_docs | LC_ALL=C sort -u) - fi - - if [[ ${#DOC_FILES[@]} -eq 0 ]]; then - echo "check-docs-links: no Markdown/MDX files to scan under $REPO_ROOT" >&2 - return 1 - fi - - log "repository root: $REPO_ROOT" - log "scope: README, CONTRIBUTING, SECURITY, CHANGELOG, docs/**/*.{md,mdx}" - if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then - log "remote: curl unique http(s) targets (disable: CHECK_DOC_LINKS_REMOTE=0 or --local-only)" - else - log "remote: skipped (local paths only)" - fi - log "Markdown file(s) (${#DOC_FILES[@]}):" - local md - for md in "${DOC_FILES[@]}"; do - case "$md" in - "$REPO_ROOT"/*) log " ${md#"$REPO_ROOT"/}" ;; - *) log " $md" ;; - esac - done - - local failures=0 - declare -a REMOTE_URLS=() - - log "phase 1/2: local file targets and Fern routes for [](url) / ![]() / (code fences skipped)" - for md in "${DOC_FILES[@]}"; do - if [[ ! -f "$md" ]]; then - echo "check-docs-links: missing file: $md" >&2 - failures=1 - continue - fi - local target rc - local _targets_output _targets_err - _targets_err="$(mktemp)" - if ! _targets_output="$(extract_targets "$md" 2>"$_targets_err")"; then - echo "check-docs-links: malformed HTML comment in $md: $(tr '\n' ' ' <"$_targets_err" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//')" >&2 - rm -f "$_targets_err" - failures=1 - continue - fi - rm -f "$_targets_err" - local line_no - while IFS=$'\t' read -r line_no target || [[ -n "${target:-}" ]]; do - [[ -z "$target" ]] && continue - if check_local_ref "$md" "$line_no" "$target"; then - rc=0 - else - rc=$? - fi - if [[ "$rc" -eq 0 ]]; then - continue - elif [[ "$rc" -eq 2 ]]; then - REMOTE_URLS+=("$target") - else - failures=1 - fi - done <<<"$_targets_output" - done - - if [[ "$failures" -ne 0 ]]; then - log "phase 1 failed" - return 1 - fi - log "phase 1 OK (local paths and Fern routes resolve)" - - local _n_raw _deduped _unique _i _u url - _n_raw="${#REMOTE_URLS[@]}" - _deduped="" - if [[ ${#REMOTE_URLS[@]} -gt 0 ]]; then - _deduped="$(printf '%s\n' "${REMOTE_URLS[@]}" | LC_ALL=C sort -u)" - _unique="$(printf '%s\n' "${REMOTE_URLS[@]}" | LC_ALL=C sort -u | grep -c . || true)" - else - _unique=0 - fi - log "http(s): ${_n_raw} reference(s) -> ${_unique} unique URL(s)" - if [[ -n "$_deduped" ]]; then - log "unique http(s) URL(s) (alphabetically):" - while IFS= read -r _u || [[ -n "${_u:-}" ]]; do - [[ -z "${_u:-}" ]] && continue - log " ${_u}" - done <<<"$_deduped" - fi - - if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then - if [[ -n "$_deduped" ]]; then - local _probe_list="" _skip_count=0 _probe_n=0 - while IFS= read -r url || [[ -n "${url:-}" ]]; do - [[ -z "${url:-}" ]] && continue - if url_should_skip_remote_probe "$url"; then - log " skipped (ignore list): ${url}" - _skip_count=$((_skip_count + 1)) - else - _probe_list+="${url}"$'\n' - fi - done <<<"$_deduped" - _probe_n="$(printf '%s\n' "$_probe_list" | grep -c . || true)" - log "phase 2/2: curl ${_probe_n} URL(s), ${_skip_count} skipped (GET, -L, fail 4xx/5xx)" - _i=0 - while IFS= read -r url || [[ -n "${url:-}" ]]; do - [[ -z "${url:-}" ]] && continue - _i=$((_i + 1)) - if [[ "$VERBOSE" -eq 1 ]]; then - log " [${_i}/${_probe_n}] ${url}" - fi - if ! check_remote_url "$url"; then - failures=1 - fi - done <<<"$_probe_list" - else - log "phase 2/2: no http(s) links" - fi - else - if [[ -n "$_deduped" ]]; then - log "phase 2/2: skipped ${_unique} URL(s) (local-only)" - else - log "phase 2/2: skipped (no http(s) links)" - fi - fi - - if [[ "$failures" -ne 0 ]]; then - log "phase 2 failed" - return 1 - fi - log "summary: ${#DOC_FILES[@]} file(s), local OK$( - [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]] && [[ ${_unique:-0} -gt 0 ]] && printf ', %s remote OK' "${_unique}" - )$( - [[ "$CHECK_DOC_LINKS_REMOTE" == 0 ]] && [[ ${_unique:-0} -gt 0 ]] && printf ' (%s remote not checked)' "${_unique}" - )" - log "done." -} - -run_links_check diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py deleted file mode 100644 index e9ea82fe44..0000000000 --- a/tests/test_docs_links.py +++ /dev/null @@ -1,184 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import platform -import shutil -import subprocess -import tempfile -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[1] -CHECK_DOCS_LINKS = REPO_ROOT / "scripts" / "check-docs-links.sh" - - -def run_link_check(file_path: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: - merged_env = os.environ.copy() - if env: - merged_env.update(env) - return subprocess.run( - ["bash", str(CHECK_DOCS_LINKS), "--local-only", str(file_path)], - cwd=REPO_ROOT, - env=merged_env, - text=True, - capture_output=True, - timeout=15, - check=False, - ) - - -@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") -def test_reports_broken_local_markdown_links_with_source_line_numbers( - tmp_path: Path, -) -> None: - md_path = tmp_path / "guide.md" - (tmp_path / "exists.md").write_text("# ok\n", encoding="utf-8") - md_path.write_text( - "\n".join( - [ - "# Guide", - "", - "[working](./exists.md)", - "[broken](./missing.md)", - "```md", - "[ignored](./inside-code-fence.md)", - "```", - "", - ] - ), - encoding="utf-8", - ) - - result = run_link_check(md_path) - output = f"{result.stdout}{result.stderr}" - - assert result.returncode == 1 - assert f"broken local link in {md_path}:4 -> ./missing.md" in output - assert "inside-code-fence.md" not in output - - -@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") -def test_ignores_links_inside_inline_code_and_html_comments(tmp_path: Path) -> None: - md_path = tmp_path / "guide.md" - md_path.write_text( - "\n".join( - [ - "# Guide", - "", - "Use `refer to [DOC PAGE](/doc/path)` as placeholder text.", - "", - "", - ] - ), - encoding="utf-8", - ) - - result = run_link_check(md_path) - output = f"{result.stdout}{result.stderr}" - - assert result.returncode == 0 - assert "/doc/path" not in output - assert "inside-comment.md" not in output - - -@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") -def test_resolves_guardrails_fern_routes(tmp_path: Path) -> None: - md_path = tmp_path / "guide.mdx" - md_path.write_text( - "\n".join( - [ - "# Guide", - "", - "[Install](/get-started/installation-guide)", - '', - "[SDK](/guardrails-python-sdk/nemoguardrails)", - "", - ] - ), - encoding="utf-8", - ) - - result = run_link_check(md_path) - output = f"{result.stdout}{result.stderr}" - - assert result.returncode == 0, output - - -@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") -def test_rejects_mdx_suffixes_for_links_that_resolve_as_fern_routes() -> None: - temp_dir = Path(tempfile.mkdtemp(prefix="check-docs-route-suffix-", dir=REPO_ROOT / "docs")) - try: - temp_path = temp_dir / "temp.mdx" - target_path = temp_dir / "target.mdx" - nav_path = temp_dir / "index.yml" - temp_nav_path = temp_path.relative_to(REPO_ROOT / "docs") - target_nav_path = target_path.relative_to(REPO_ROOT / "docs") - temp_path.write_text( - "\n".join( - [ - "---", - 'title: "Temporary Link Check Page"', - "---", - "", - "[Wrong](target.mdx)", - "[Right](target)", - "", - ] - ), - encoding="utf-8", - ) - target_path.write_text("# Target\n", encoding="utf-8") - nav_path.write_text( - "\n".join( - [ - "navigation:", - ' - page: "Temp"', - f" path: {temp_nav_path}", - " slug: temp", - ' - page: "Target"', - f" path: {target_nav_path}", - " slug: target", - "", - ] - ), - encoding="utf-8", - ) - - result = run_link_check(temp_path, {"CHECK_DOCS_FERN_NAV_YML": str(nav_path)}) - output = f"{result.stdout}{result.stderr}" - - assert result.returncode == 1 - assert (f"route-style link should omit .md/.mdx extension in {temp_path}:5 -> target.mdx") in output - assert f"broken local link in {temp_path}:6 -> target" not in output - finally: - shutil.rmtree(temp_dir) - - -@pytest.mark.skipif(platform.system() == "Windows", reason="bash script not available on Windows") -def test_fails_loudly_on_malformed_html_comments(tmp_path: Path) -> None: - md_path = tmp_path / "guide.md" - md_path.write_text( - "\n".join(["# Guide", "