Sync with upstream develop

.github/workflows/docs-build.yaml

@@ -45,7 +45,7 @@ jobs:
             - name: Configure poetry
               run: poetry config virtualenvs.in-project true
             - name: Install dependencies
-              run: poetry install --only docs
+              run: poetry install --with docs
             - name: Build documentation
               run: make docs
             - name: Delete unnecessary files

.gitignore

@@ -57,6 +57,9 @@ local_cache
 docs/user_guides/llm/vertexai/config
 docs/**/config
 
+# Ignore auto-generated API reference stubs
+docs/reference/python-api/generated/
+
 # Ignoring this for now
 # /scripts
 

docs/_templates/autosummary/class.rst

@@ -0,0 +1,34 @@
+{{ fullname | escape | underline }}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :inherited-members:
+
+   {% block methods %}
+   {% if methods %}
+   .. rubric:: Methods
+
+   .. autosummary::
+      :nosignatures:
+
+   {% for item in methods %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: Attributes
+
+   .. autosummary::
+
+   {% for item in attributes %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

docs/_templates/autosummary/module.rst

@@ -0,0 +1,46 @@
+{{ fullname | escape | underline }}
+
+.. automodule:: {{ fullname }}
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+   {% block classes %}
+   {% if classes %}
+   .. rubric:: Classes
+
+   .. autosummary::
+      :toctree:
+      :template: class.rst
+
+   {% for item in classes %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block functions %}
+   {% if functions %}
+   .. rubric:: Functions
+
+   .. autosummary::
+      :toctree:
+
+   {% for item in functions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block exceptions %}
+   {% if exceptions %}
+   .. rubric:: Exceptions
+
+   .. autosummary::
+      :toctree:
+
+   {% for item in exceptions %}
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

docs/conf.py

@@ -24,6 +24,9 @@
 # Add local extensions to path
 sys.path.insert(0, str(Path(__file__).parent / "_extensions"))
 
+# Add the project root to path so autodoc can import nemoguardrails
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
 project = "NVIDIA NeMo Guardrails Library Developer Guide"
 this_year = date.today().year
 copyright = f"2023-{this_year}, NVIDIA Corporation"
@@ -35,7 +38,12 @@
 
 extensions = [
     "myst_parser",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
     "sphinx.ext.intersphinx",
+    "sphinx_autodoc_typehints",
     "sphinx_copybutton",
     "sphinx_reredirects",
     "sphinx_design",
@@ -44,6 +52,34 @@
     "search_assets",  # Enhanced search assets extension
 ]
 
+# -- Autodoc configuration ---------------------------------------------------
+autodoc_default_options = {
+    "members": True,
+    "undoc-members": False,
+    "show-inheritance": True,
+    "member-order": "bysource",
+}
+autodoc_typehints = "description"
+autodoc_class_signature = "separated"
+
+# Mock imports for optional dependencies that may not be installed
+autodoc_mock_imports = [
+    "presidio_analyzer",
+    "presidio_anonymizer",
+    "spacy",
+    "google.cloud",
+    "yara",
+    "fast_langdetect",
+    "opentelemetry",
+    "streamlit",
+    "tqdm",
+]
+
+# -- Autosummary configuration -----------------------------------------------
+autosummary_generate = True
+autosummary_generate_overwrite = True
+autosummary_imported_members = True
+
 redirects = {
     "introduction": "index.html",
     "documentation": "index.html",

docs/configure-rails/yaml-schema/streaming/global-streaming.md

@@ -110,9 +110,11 @@ nemoguardrails chat path/to/config --streaming
 
 ---
 
-## Token Usage Tracking
+## Streaming Metadata
+
+Use `include_metadata=True` in `stream_async()` to receive per-chunk metadata (token usage, finish reason). See [Streaming Metadata](../../../run-rails/using-python-apis/streaming.md#streaming-metadata) for details.
 
-When using `stream_async()`, NeMo Guardrails automatically enables token usage tracking by setting `stream_usage = True` on the underlying LLM model.
+## Token Usage Tracking
 
 Access token usage through the `log` generation option: