Dynamic OpenAI SUTs (#1131)

* move openai to core

* openai factory

* fix openai compliance annotator

* Remove get_secrets from factory interface

* remove openai plugin dep from anthropic

Author: bkorycki

plugins/anthropic/pyproject.toml

@@ -9,7 +9,6 @@ packages = [{include = "modelgauge"}]
 [tool.poetry.dependencies]
 python = "^3.10"
 anthropic = "*"
-modelgauge_openai = {version = "*", optional = false}
 
 
 [build-system]

plugins/openai/README.md

@@ -80,7 +80,6 @@ modelgauge_azure = {path = "plugins/azure", develop = true, optional=true}
 modelgauge_baseten = {path = "plugins/baseten", develop = true, optional=true}
 modelgauge_demo_plugin = {path = "demo_plugin", develop = true, optional=false}
 modelgauge_nvidia = {path = "plugins/nvidia", develop = true, optional=true}
-modelgauge_openai = {path = "plugins/openai", develop = true, optional=true}
 modelgauge_perspective_api = {path = "plugins/perspective_api", develop = true, optional=true}
 modelgauge_google = {path = "plugins/google", develop = true, optional=true}
 modelgauge_vertexai = {path = "plugins/vertexai", develop = true, optional = true}
@@ -90,6 +89,7 @@ modelgauge_amazon = {path = "plugins/amazon", develop = true, optional=true}
 prometheus-client = "^0.21.1"
 llama-api-client = "^0.1.1"
 huggingface-hub = "^0.30.2"
+openai = "^1.8.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest-datafiles = "^3.0.0"
@@ -109,13 +109,12 @@ azure = ["modelgauge_azure"]
 baseten = ["modelgauge_baseten"]
 demo = ["modelgauge_demo_plugin"]
 nvidia = ["modelgauge_nvidia"]
-openai = ["modelgauge_openai"]
 perspective_api = ["modelgauge_perspective_api"]
 google = ["modelgauge_google"]
 vertexai = ["modelgauge_vertexai"]
 mistral = ["modelgauge_mistral"]
 amazon = ["modelgauge_amazon"]
-all_plugins = ["modelgauge_anthropic", "modelgauge_azure", "modelgauge_baseten", "modelgauge_demo_plugin", "modelgauge_nvidia", "modelgauge_openai", "modelgauge_perspective_api", "modelgauge_google", "modelgauge_vertexai", "modelgauge_mistral", "modelgauge_amazon"]
+all_plugins = ["modelgauge_anthropic", "modelgauge_azure", "modelgauge_baseten", "modelgauge_demo_plugin", "modelgauge_nvidia", "modelgauge_perspective_api", "modelgauge_google", "modelgauge_vertexai", "modelgauge_mistral", "modelgauge_amazon"]
 
 [tool.poetry.scripts]
 modelbench = "modelbench.run:cli"
@@ -143,7 +142,7 @@ line-length = 120
 # However, by default mypy uses those to map file paths to modules. This override fixes that.
 # https://mypy.readthedocs.io/en/stable/config_file.html#confval-explicit_package_bases
 explicit_package_bases = true
-mypy_path = "., demo_plugin, plugins/openai, plugins/perspective_api, src, tests"
+mypy_path = "., demo_plugin, plugins/perspective_api, src, tests"
 
 [[tool.mypy.overrides]]
 module = "modelgauge.tests.*,modelgauge.annotators.*,modelgauge.safety_model_response,plugins.*"

plugins/openai/pyproject.toml

@@ -41,11 +41,8 @@ def __init__(
         self.decoder = _DEFAULT_MAPPING if decoder is None else decoder
 
     def translate_request(self, test_item: TestItem, response: SUTResponse):
-        openai_prompt = TextPrompt(
-            text=self.formatter(test_item, response),
-            options=SUTOptions(temperature=0),
-        )
-        return self.model.translate_text_prompt(openai_prompt)
+        openai_prompt = TextPrompt(text=self.formatter(test_item, response))
+        return self.model.translate_text_prompt(openai_prompt, options=SUTOptions(temperature=0))
 
     def annotate(self, annotation_request):
         """Returns an annotation for a single TestItem's interactions."""

poetry.lock

@@ -27,12 +27,6 @@ class UnknownSUTMakerError(Exception):
 
 
 class DynamicSUTFactory(ABC):
-
-    @staticmethod
-    @abstractmethod
-    def get_secrets() -> InjectSecret:
-        pass
-
     @staticmethod
     @abstractmethod
     def make_sut(sut_metadata: DynamicSUTMetadata):

pyproject.toml

@@ -11,13 +11,19 @@
 
 from modelgauge.dynamic_sut_factory import UnknownSUTMakerError
 from modelgauge.dynamic_sut_metadata import DynamicSUTMetadata
+from modelgauge.suts.openai_sut_factory import OpenAISUTFactory
 from modelgauge.suts.together_sut_factory import TogetherSUTFactory
 
 # Maps a string to the module and factory function in that module
 # that can be used to create a dynamic sut
 DYNAMIC_SUT_FACTORIES: dict = {
     "proxied": {"hfrelay": hsf.make_sut},
-    "direct": {"together": TogetherSUTFactory.make_sut, "huggingface": hsf.make_sut, "hf": hsf.make_sut},
+    "direct": {
+        "openai": OpenAISUTFactory.make_sut,
+        "together": TogetherSUTFactory.make_sut,
+        "huggingface": hsf.make_sut,
+        "hf": hsf.make_sut,
+    },
 }
 
 

…nnotators/openai_compliance_annotator.py …nnotators/openai_compliance_annotator.pyplugins/openai/modelgauge/annotators/openai_compliance_annotator.py renamed to src/modelgauge/annotators/openai_compliance_annotator.py plugins/openai/modelgauge/annotators/openai_compliance_annotator.py renamed to src/modelgauge/annotators/openai_compliance_annotator.py

@@ -0,0 +1,40 @@
+from openai import OpenAI, NotFoundError
+
+from modelgauge.config import load_secrets_from_config
+from modelgauge.dynamic_sut_factory import DynamicSUTFactory, ModelNotSupportedError
+from modelgauge.dynamic_sut_metadata import DynamicSUTMetadata
+from modelgauge.secret_values import InjectSecret
+from modelgauge.suts.openai_client import OpenAIApiKey, OpenAIChat, OpenAIOrgId
+
+
+DRIVER_NAME = "openai"
+
+
+class OpenAISUTFactory(DynamicSUTFactory):
+    @staticmethod
+    def _model_exists(sut_metadata: DynamicSUTMetadata):
+        secrets = load_secrets_from_config()
+        api_key = OpenAIApiKey.make(secrets).value
+        org_id = OpenAIOrgId.make(secrets).value
+
+        client = OpenAI(api_key=api_key, organization=org_id, max_retries=7)
+
+        try:
+            client.models.retrieve(sut_metadata.model)
+        except NotFoundError:
+            return False
+        return True
+
+    @staticmethod
+    def make_sut(sut_metadata: DynamicSUTMetadata):
+        if not OpenAISUTFactory._model_exists(sut_metadata):
+            raise ModelNotSupportedError(f"Model {sut_metadata.model} not found or not available on openai.")
+
+        assert sut_metadata.driver == DRIVER_NAME
+        return (
+            OpenAIChat,
+            DynamicSUTMetadata.make_sut_uid(sut_metadata),
+            sut_metadata.model,
+            InjectSecret(OpenAIApiKey),
+            InjectSecret(OpenAIOrgId),
+        )