Update provider spec & upgrade to llama-stack 0.2.22

.vscode/launch.json

@@ -6,11 +6,21 @@
     "configurations": [
 
         {
-            "name": "Debug Ragas Distribution",
+            "name": "Debug Ragas Distribution -- Remote",
             "type": "debugpy",
             "request": "launch",
             "module": "llama_stack.cli.llama",
-            "args": ["stack", "run", "distribution/run.yaml"],
+            "args": ["stack", "run", "distribution/run-remote.yaml"],
+            "cwd": "${workspaceFolder}",
+            "envFile": "${workspaceFolder}/.env",
+            "justMyCode": false
+        },
+        {
+            "name": "Debug Ragas Distribution -- Inline",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "llama_stack.cli.llama",
+            "args": ["stack", "run", "distribution/run-inline.yaml"],
             "cwd": "${workspaceFolder}",
             "envFile": "${workspaceFolder}/.env",
             "justMyCode": false

README.md

@@ -38,19 +38,7 @@ There are two versions of the provider:
     ```
 - The sample LS distributions (one for inline and one for remote provider) is a simple LS distribution that uses Ollama for inference and embeddings. See the provider-specific sections below for setup and run commands.
 
-### Inline provider
-
-Create a `.env` file with the required environment variable:
-```bash
-EMBEDDING_MODEL=all-MiniLM-L6-v2
-```
-
-Run the server:
-```bash
-dotenv run uv run llama stack run distribution/run-inline.yaml
-```
-
-### Remote provider
+### Remote provider (default)
 
 Create a `.env` file with the following:
 ```bash
@@ -75,6 +63,19 @@ Run the server:
 dotenv run uv run llama stack run distribution/run-remote.yaml
 ```
 
+### Inline provider (need to specify `.inline` in the module name)
+
+Create a `.env` file with the required environment variable:
+```bash
+EMBEDDING_MODEL=all-MiniLM-L6-v2
+```
+
+Run the server:
+```bash
+dotenv run uv run llama stack run distribution/run-inline.yaml
+```
+
+You will notice that `run-inline.yaml` file has the module name as `llama_stack_provider_ragas.inline`, in order to specify the inline provider.
 
 ## Usage
 See the demos in the `demos` directory.

distribution/run-inline.yaml

@@ -5,10 +5,13 @@ apis:
   - inference
   - telemetry
   - datasetio
+  - files
+  - benchmarks
 providers:
   eval:
-    - provider_id: trustyai_ragas_inline
+    - provider_id: trustyai_ragas
       provider_type: inline::trustyai_ragas
+      module: llama_stack_provider_ragas.inline
       config:
         embedding_model: ${env.EMBEDDING_MODEL}
   datasetio:
@@ -18,19 +21,29 @@ providers:
         kvstore:
           type: sqlite
           namespace: null
-          db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/trustyai_ragas_distro}/localfs_datasetio.db
+          db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/trustyai_ragas_distro_inline}/localfs_datasetio.db
   inference:
     - provider_id: ollama
       provider_type: remote::ollama
       config:
-        url: ${env.OLLAMA_URL:http://localhost:11434}
+        url: ${env.OLLAMA_URL:=http://localhost:11434}
   telemetry:
     - provider_id: meta-reference
       provider_type: inline::meta-reference
       config:
-        service_name: ${env.OTEL_SERVICE_NAME:}
-      sinks: ${env.TELEMETRY_SINKS:console,sqlite}
-      sqlite_db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/trustyai_ragas_distro}/trace_store.db
+        service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
+        sinks: ${env.TELEMETRY_SINKS:=console,sqlite}
+        sqlite_db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/trustyai_ragas_distro_inline}/trace_store.db
+        otel_exporter_otlp_endpoint: ${env.OTEL_EXPORTER_OTLP_ENDPOINT:=}
+  files:
+    - provider_id: meta-reference-files
+      provider_type: inline::localfs
+      config:
+        storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/trustyai_ragas_distro_inline/files}
+        metadata_store:
+          type: sqlite
+          db_path: ${env.METADATA_STORE_DB_PATH:=~/.llama/distributions/trustyai_ragas_distro_inline}/registry.db}
+
 models:
   - metadata:
       embedding_dimension: 384
@@ -46,4 +59,3 @@ models:
 server:
   host: localhost
   port: 8321
-external_providers_dir: ./providers.d

distribution/run-remote.yaml

@@ -3,12 +3,15 @@ image_name: trustyai_ragas_distro_remote
 apis:
   - eval
   - inference
+  - files
+  - benchmarks
   - telemetry
   - datasetio
 providers:
   eval:
-    - provider_id: trustyai_ragas_remote
+    - provider_id: trustyai_ragas
       provider_type: remote::trustyai_ragas
+      module: llama_stack_provider_ragas
       config:
         embedding_model: ${env.EMBEDDING_MODEL}
         kubeflow_config:
@@ -25,19 +28,28 @@ providers:
         kvstore:
           type: sqlite
           namespace: null
-          db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/trustyai_ragas_distro}/localfs_datasetio.db
+          db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/trustyai_ragas_distro_remote}/localfs_datasetio.db
   inference:
     - provider_id: ollama
       provider_type: remote::ollama
       config:
-        url: ${env.OLLAMA_URL:http://localhost:11434}
+        url: ${env.OLLAMA_URL:=http://localhost:11434}
   telemetry:
     - provider_id: meta-reference
       provider_type: inline::meta-reference
       config:
-        service_name: ${env.OTEL_SERVICE_NAME:}
-      sinks: ${env.TELEMETRY_SINKS:console,sqlite}
-      sqlite_db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/trustyai_ragas_distro}/trace_store.db
+        service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
+        sinks: ${env.TELEMETRY_SINKS:=console,sqlite}
+        sqlite_db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/trustyai_ragas_distro_remote}/trace_store.db
+        otel_exporter_otlp_endpoint: ${env.OTEL_EXPORTER_OTLP_ENDPOINT:=}
+  files:
+    - provider_id: meta-reference-files
+      provider_type: inline::localfs
+      config:
+        storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/trustyai_ragas_distro_remote/files}
+        metadata_store:
+          type: sqlite
+          db_path: ${env.METADATA_STORE_DB_PATH:=~/.llama/distributions/trustyai_ragas_distro_remote}/registry.db}
 models:
   - metadata:
       embedding_dimension: 384
@@ -53,4 +65,3 @@ models:
 server:
   host: localhost
   port: 8321
-external_providers_dir: ./providers.d

docs/modules/ROOT/pages/index.adoc

@@ -13,8 +13,8 @@ The goal is to provide all of Ragas' evaluation functionality over Llama Stack's
 
 There are two versions of the provider:
 
+* `remote`: runs the Ragas evaluation in a remote process, using Kubeflow Pipelines. This is the *default* when using the module-based import.
 * `inline`: runs the Ragas evaluation in the same process as the Llama Stack server.
-* `remote`: runs the Ragas evaluation in a remote process, using Kubeflow Pipelines.
 
 == Getting Started
 

docs/modules/ROOT/pages/inline/configuration.adoc

@@ -23,8 +23,9 @@ The inline provider is setup in the following lines of the `run-inline.yaml`:
 [,yaml]
 ----
 eval:
-  - provider_id: trustyai_ragas_inline
+  - provider_id: trustyai_ragas
     provider_type: inline::trustyai_ragas
+    module: llama_stack_provider_ragas.inline
     config:
       embedding_model: ${env.EMBEDDING_MODEL}
 ----

docs/modules/ROOT/pages/remote/configuration.adoc

@@ -54,8 +54,9 @@ The remote provider is setup in the following lines of the `run-remote.yaml`:
 [,yaml]
 ----
 eval:
-  - provider_id: trustyai_ragas_remote
+  - provider_id: trustyai_ragas
     provider_type: remote::trustyai_ragas
+    module: llama_stack_provider_ragas.remote
     config:
       embedding_model: ${env.EMBEDDING_MODEL}
       kubeflow_config:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "llama-stack-provider-ragas"
-version = "0.1.1"
+version = "0.2.3"
 description = "Ragas evaluation as an out-of-tree Llama Stack provider"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -25,7 +25,8 @@ authors = [
 keywords = ["llama-stack", "ragas", "evaluation"]
 dependencies = [
     "setuptools-scm",
-    "llama-stack==0.2.11",
+    "llama-stack==0.2.22",
+    "greenlet==3.2.4", # inline/files/localfs errors saying greenlet not found
     "ragas==0.3.0",
     "pandas==2.3.0",
     "pyarrow==20.0.0",

src/llama_stack_provider_ragas/__init__.py

@@ -1,27 +0,0 @@
-from typing import Any
-
-from llama_stack.distribution.datatypes import Api
-
-from .config import RagasProviderInlineConfig, RagasProviderRemoteConfig
-from .eval_inline import RagasEvaluatorInline
-from .eval_remote import RagasEvaluatorRemote
-
-
-async def get_adapter_impl(
-    config: RagasProviderRemoteConfig,
-    deps: dict[Api, Any],
-) -> RagasEvaluatorRemote:
-    return RagasEvaluatorRemote(config)
-
-
-async def get_provider_impl(
-    config: RagasProviderInlineConfig,
-    deps: dict[Api, Any],
-) -> RagasEvaluatorInline:
-    return RagasEvaluatorInline(config, deps[Api.datasetio], deps[Api.inference])
-
-
-__all__ = [
-    "get_adapter_impl",
-    "get_provider_impl",
-]

src/llama_stack_provider_ragas/inline/__init__.py

@@ -0,0 +1,16 @@
+from typing import Any
+
+from llama_stack.apis.datatypes import Api
+
+from ..config import RagasProviderInlineConfig
+from .ragas_inline_eval import RagasEvaluatorInline
+
+
+async def get_provider_impl(
+    config: RagasProviderInlineConfig,
+    deps: dict[Api, Any],
+) -> RagasEvaluatorInline:
+    return RagasEvaluatorInline(config, deps[Api.datasetio], deps[Api.inference])
+
+
+__all__ = ["RagasEvaluatorInline", "get_provider_impl"]

src/llama_stack_provider_ragas/inline/provider.py

@@ -0,0 +1,18 @@
+from llama_stack.providers.datatypes import Api, InlineProviderSpec, ProviderSpec
+
+
+def get_provider_spec() -> ProviderSpec:
+    return InlineProviderSpec(
+        api=Api.eval,
+        provider_type="inline::trustyai_ragas",
+        pip_packages=["ragas==0.3.0"],
+        config_class="llama_stack_provider_ragas.config.RagasProviderInlineConfig",
+        module="llama_stack_provider_ragas.inline",
+        api_dependencies=[
+            Api.inference,
+            Api.files,
+            Api.benchmarks,
+            Api.datasetio,
+            Api.telemetry,
+        ],
+    )

src/llama_stack_provider_ragas/eval_inline.py → src/llama_stack_provider_ragas/inline/ragas_inline_eval.py

@@ -22,10 +22,10 @@
 )
 from ragas.run_config import RunConfig
 
-from .config import RagasProviderInlineConfig
-from .constants import METRIC_MAPPING
-from .errors import RagasEvaluationError
-from .logging_utils import render_dataframe_as_table
+from ..config import RagasProviderInlineConfig
+from ..constants import METRIC_MAPPING
+from ..errors import RagasEvaluationError
+from ..logging_utils import render_dataframe_as_table
 from .wrappers_inline import LlamaStackInlineEmbeddings, LlamaStackInlineLLM
 
 logger = logging.getLogger(__name__)

src/llama_stack_provider_ragas/provider.py

@@ -1,18 +1,5 @@
-from llama_stack.providers.datatypes import (
-    AdapterSpec,
-    Api,
-    ProviderSpec,
-    remote_provider_spec,
-)
+# remote is the default provider
+from .remote import get_adapter_impl
+from .remote.provider import get_provider_spec
 
-
-def get_provider_spec() -> ProviderSpec:
-    return remote_provider_spec(
-        api=Api.eval,
-        adapter=AdapterSpec(
-            adapter_type="trustyai_ragas",
-            pip_packages=["ragas"],  # ["datasets", "langchain-core"],
-            config_class="config.RagasEvalProviderConfig",
-            module="ragas_eval",
-        ),
-    )
+__all__ = ["get_provider_spec", "get_adapter_impl"]

src/llama_stack_provider_ragas/remote/__init__.py

@@ -0,0 +1,16 @@
+from typing import Any
+
+from llama_stack.apis.datatypes import Api
+
+from ..config import RagasProviderRemoteConfig
+from .ragas_remote_eval import RagasEvaluatorRemote
+
+
+async def get_adapter_impl(
+    config: RagasProviderRemoteConfig,
+    deps: dict[Api, Any],
+) -> RagasEvaluatorRemote:
+    return RagasEvaluatorRemote(config)
+
+
+__all__ = ["RagasEvaluatorRemote", "get_adapter_impl"]

src/llama_stack_provider_ragas/kubeflow/components.py → src/llama_stack_provider_ragas/remote/kubeflow/components.py

@@ -42,7 +42,7 @@ def run_ragas_evaluation(
 
     from llama_stack_provider_ragas.constants import METRIC_MAPPING
     from llama_stack_provider_ragas.logging_utils import render_dataframe_as_table
-    from llama_stack_provider_ragas.wrappers_remote import (
+    from llama_stack_provider_ragas.remote.wrappers_remote import (
         LlamaStackRemoteEmbeddings,
         LlamaStackRemoteLLM,
     )