Merge pull request lightspeed-core#422 from tisnik/lcore-437-check-llama-stack-version-on-startup--

LCORE-437: check Llama Stack version on startup

Author: tisnik

README.md

@@ -27,6 +27,7 @@ The service includes comprehensive user data collection capabilities for various
         * [Llama Stack project and configuration](#llama-stack-project-and-configuration)
         * [Check connection to Llama Stack](#check-connection-to-llama-stack)
     * [Llama Stack as client library](#llama-stack-as-client-library)
+    * [Llama Stack version check](#llama-stack-version-check)
     * [User data collection](#user-data-collection)
     * [System prompt](#system-prompt)
     * [Safety Shields](#safety-shields)
@@ -243,6 +244,12 @@ user_data_collection:
   transcripts_storage: "/tmp/data/transcripts"
 ```
 
+## Llama Stack version check
+
+During Lightspeed Core Stack service startup, the Llama Stack version is retrieved. The version is tested against two constants `MINIMAL_SUPPORTED_LLAMA_STACK_VERSION` and `MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION` which are defined in `src/constants.py`. If the actual Llama Stack version is outside the range defined by these two constants, the service won't start and administrator will be informed about this problem.
+
+
+
 ## User data collection
 
 The Lightspeed Core Stack includes comprehensive user data collection capabilities to gather various types of user interaction data for analysis and improvement. This includes feedback, conversation transcripts, and other user interaction data.

pyproject.toml

@@ -36,6 +36,7 @@ dependencies = [
     "email-validator>=2.2.0",
     "openai==1.99.9",
     "sqlalchemy>=2.0.42",
+    "semver<4.0.0",
 ]
 
 
@@ -91,6 +92,7 @@ dev = [
     "build>=1.2.2.post1",
     "twine>=6.1.0",
     "openapi-to-md>=0.1.0b2",
+    "pytest-subtests>=0.14.2",
 ]
 llslibdev = [
     # To check llama-stack API provider dependecies:

src/constants.py

@@ -1,5 +1,9 @@
 """Constants used in business logic."""
 
+# Minimal and maximal supported Llama Stack version
+MINIMAL_SUPPORTED_LLAMA_STACK_VERSION = "0.2.17"
+MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION = "0.2.17"
+
 UNABLE_TO_PROCESS_RESPONSE = "Unable to process this request"
 
 # Supported attachment types

src/lightspeed_stack.py

@@ -12,6 +12,7 @@
 from runners.uvicorn import start_uvicorn
 from configuration import configuration
 from client import AsyncLlamaStackClientHolder
+from utils.llama_stack_version import check_llama_stack_version
 
 FORMAT = "%(message)s"
 logging.basicConfig(
@@ -66,6 +67,8 @@ def main() -> None:
     asyncio.run(
         AsyncLlamaStackClientHolder().load(configuration.configuration.llama_stack)
     )
+    client = AsyncLlamaStackClientHolder().get_client()
+    asyncio.run(check_llama_stack_version(client))
 
     if args.dump_configuration:
         configuration.configuration.dump()

src/utils/llama_stack_version.py

@@ -0,0 +1,51 @@
+"""Check if the Llama Stack version is supported by the LCS."""
+
+import logging
+
+from semver import Version
+
+from llama_stack_client._client import AsyncLlamaStackClient
+
+
+from constants import (
+    MINIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+    MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+)
+
+logger = logging.getLogger("utils.llama_stack_version")
+
+
+class InvalidLlamaStackVersionException(Exception):
+    """Llama Stack version is not valid."""
+
+
+async def check_llama_stack_version(
+    client: AsyncLlamaStackClient,
+) -> None:
+    """Check if the Llama Stack version is supported by the LCS."""
+    version_info = await client.inspect.version()
+    compare_versions(
+        version_info.version,
+        MINIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+        MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+    )
+
+
+def compare_versions(version_info: str, minimal: str, maximal: str) -> None:
+    """Compare current Llama Stack version with minimal and maximal allowed versions."""
+    current_version = Version.parse(version_info)
+    minimal_version = Version.parse(minimal)
+    maximal_version = Version.parse(maximal)
+    logger.debug("Current version: %s", current_version)
+    logger.debug("Minimal version: %s", minimal_version)
+    logger.debug("Maximal version: %s", maximal_version)
+
+    if current_version < minimal_version:
+        raise InvalidLlamaStackVersionException(
+            f"Llama Stack version >= {minimal_version} is required, but {current_version} is used"
+        )
+    if current_version > maximal_version:
+        raise InvalidLlamaStackVersionException(
+            f"Llama Stack version <= {maximal_version} is required, but {current_version} is used"
+        )
+    logger.info("Correct Llama Stack version : %s", current_version)

tests/unit/utils/test_llama_stack_version.py

@@ -0,0 +1,101 @@
+"""Unit tests for utility function to check Llama Stack version."""
+
+import pytest
+from semver import Version
+
+from llama_stack_client.types import VersionInfo
+
+from utils.llama_stack_version import (
+    check_llama_stack_version,
+    InvalidLlamaStackVersionException,
+)
+
+from constants import (
+    MINIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+    MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION,
+)
+
+
+@pytest.mark.asyncio
+async def test_check_llama_stack_version_minimal_supported_version(mocker):
+    """Test the check_llama_stack_version function."""
+
+    # mock the Llama Stack client
+    mock_client = mocker.AsyncMock()
+    mock_client.inspect.version.return_value = VersionInfo(
+        version=MINIMAL_SUPPORTED_LLAMA_STACK_VERSION
+    )
+
+    # test if the version is checked
+    await check_llama_stack_version(mock_client)
+
+
+@pytest.mark.asyncio
+async def test_check_llama_stack_version_maximal_supported_version(mocker):
+    """Test the check_llama_stack_version function."""
+
+    # mock the Llama Stack client
+    mock_client = mocker.AsyncMock()
+    mock_client.inspect.version.return_value = VersionInfo(
+        version=MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION
+    )
+
+    # test if the version is checked
+    await check_llama_stack_version(mock_client)
+
+
+@pytest.mark.asyncio
+async def test_check_llama_stack_version_too_small_version(mocker):
+    """Test the check_llama_stack_version function."""
+
+    # mock the Llama Stack client
+    mock_client = mocker.AsyncMock()
+
+    # that is surely out of range
+    mock_client.inspect.version.return_value = VersionInfo(version="0.0.0")
+
+    expected_exception_msg = (
+        f"Llama Stack version >= {MINIMAL_SUPPORTED_LLAMA_STACK_VERSION} "
+        + "is required, but 0.0.0 is used"
+    )
+    # test if the version is checked
+    with pytest.raises(InvalidLlamaStackVersionException, match=expected_exception_msg):
+        await check_llama_stack_version(mock_client)
+
+
+async def _check_version_must_fail(mock_client, bigger_version):
+    mock_client.inspect.version.return_value = VersionInfo(version=str(bigger_version))
+
+    expected_exception_msg = (
+        f"Llama Stack version <= {MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION} is required, "
+        + f"but {bigger_version} is used"
+    )
+    # test if the version is checked
+    with pytest.raises(InvalidLlamaStackVersionException, match=expected_exception_msg):
+        await check_llama_stack_version(mock_client)
+
+
+@pytest.mark.asyncio
+async def test_check_llama_stack_version_too_big_version(mocker, subtests):
+    """Test the check_llama_stack_version function."""
+
+    # mock the Llama Stack client
+    mock_client = mocker.AsyncMock()
+
+    max_version = Version.parse(MAXIMAL_SUPPORTED_LLAMA_STACK_VERSION)
+
+    with subtests.test(msg="Increased patch number"):
+        bigger_version = max_version.bump_patch()
+        await _check_version_must_fail(mock_client, bigger_version)
+
+    with subtests.test(msg="Increased minor number"):
+        bigger_version = max_version.bump_minor()
+        await _check_version_must_fail(mock_client, bigger_version)
+
+    with subtests.test(msg="Increased major number"):
+        bigger_version = max_version.bump_major()
+        await _check_version_must_fail(mock_client, bigger_version)
+
+    with subtests.test(msg="Increased all numbers"):
+        bigger_version = max_version.bump_major().bump_minor().bump_patch()
+        await _check_version_must_fail(mock_client, bigger_version)