Merge pull request #1237 from PacificAI/release/2.8.0

partial implementation from release/2.8.0

Author: chakravarthik27

.github/workflows/build_and_test.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9","3.10", "3.11" ]
+        python-version: ["3.12", "3.13" ]
 
     steps:
       - name: Free up disk space at start
@@ -53,7 +53,7 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry cache clear pypi --all -n > /dev/null
-          poetry install --with dev --all-extras --no-cache --quiet --no-interaction
+          poetry install --with dev --all-extras --no-cache --no-interaction
           source ./.venv/bin/activate && pip uninstall -y pyspark && rm -rf ./.venv/lib/python${{ matrix.python-version }}/site-packages/pyspark*/
           pip install pyspark==3.5.6
 

.github/workflows/llm_tests_build.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10" ]
+        python-version: [ "3.12", "3.13" ]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/release.yaml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: [3.12]
         poetry-version: [2.1.3]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}

langtest/datahandler/datasource.py

@@ -808,7 +808,7 @@ def load_raw_data(self, standardize_columns: bool = False) -> List[Dict]:
                 parsed CSV file into list of dicts
         """
 
-        if type(self._file_path) == dict:
+        if isinstance(self._file_path, dict):
             df = pd.read_csv(self._file_path["data_source"])
 
             if self.task == "text-classification":

langtest/embeddings/openai.py

@@ -10,15 +10,15 @@
 class OpenaiEmbeddings:
     LIB_NAME = "openai"
 
-    def __init__(self, model="text-embedding-ada-002"):
+    def __init__(self, model="text-embedding-3-small"):
         self.model = model
         self.api_key = os.environ.get("OPENAI_API_KEY")
         self.openai = None
         self._check_openai_package()
         if not self.api_key:
             raise ValueError(Errors.E032())
 
-        self.openai.api_key = self.api_key
+        # self.openai.api_key = self.api_key
 
     def _check_openai_package(self):
         """Check if the 'openai' package is installed and import the required functions.
@@ -44,13 +44,17 @@ def get_embedding(
             list[float]: A list of floating-point values representing the text's embedding.
         """
         if isinstance(text, list):
-            response = self.openai.Embedding.create(input=text, model=self.model)
+            response = self.openai.Client(api_key=self.api_key).embedding.create(
+                input=text, model=self.model
+            )
             embedding = [
-                np.array(response["data"][i]["embedding"]).reshape(1, -1)
+                np.array(response.data[i].embedding).reshape(1, -1)
                 for i in range(len(text))
             ]
             return embedding
         else:
-            response = self.openai.Embedding.create(input=[text], model=self.model)
-            embedding = np.array(response["data"][0]["embedding"]).reshape(1, -1)
+            response = self.openai.Client(api_key=self.api_key).embedding.create(
+                input=[text], model=self.model
+            )
+            embedding = np.array(response.data[0].embedding).reshape(1, -1)
             return embedding

langtest/langtest.py

@@ -286,7 +286,7 @@ def configure(self, config: Union[HarnessConfig, dict, str]) -> HarnessConfig:
         Returns:
             dict: Loaded configuration.
         """
-        if type(config) == dict:
+        if isinstance(config, dict):
             self._config = config
         else:
             with open(config, "r", encoding="utf-8") as yml:

langtest/metrics/llm_eval.py

@@ -66,25 +66,22 @@ def build_prompt(
                 f"""\n\nScore the student answer based on the following criteria:\n{eval_criteria}"""
             )
 
-        prompt += dedent(
-            f"""
-        Example Format:
-        QUESTION: question here
-        STUDENT ANSWER: student's answer here
-        TRUE ANSWER: true answer here
-        GRADE: {grade_list} here
-
-        {
-            ("Grade the student answers based ONLY on their factual accuracy. Ignore differences"
-             " in punctuation and phrasing between the student answer and true answer. It is OK "
-             "if the student answer contains more or relevant information than the true answer, as"
-             " long as it does not contain any conflicting statements. Begin!")
-        }
-
-        QUESTION: {{query}}
-        STUDENT ANSWER: {{result}}
-        TRUE ANSWER: {{answer}}
-        GRADE:"""
+        prompt += (
+            "Example Format:\n"
+            "QUESTION: question here\n"
+            "STUDENT ANSWER: student's answer here\n"
+            "TRUE ANSWER: true answer here\n"
+            f"GRADE: {grade_list} here"
+            "\n\n"
+            "Grade the student answers based ONLY on their factual accuracy. Ignore differences"
+            " in punctuation and phrasing between the student answer and true answer. It is OK "
+            "if the student answer contains more or relevant information than the true answer, as"
+            " long as it does not contain any conflicting statements. Begin!"
+            "\n\n"
+            "QUESTION: {{query}}\n"
+            "STUDENT ANSWER: {{result}}\n"
+            "TRUE ANSWER: {{answer}}\n"
+            "GRADE:\n"
         )
         return prompt
 

langtest/modelhandler/__init__.py

@@ -33,15 +33,15 @@
 
 
 if "langchain" in INSTALLED_HUBS:
-    import langchain
+    import langchain_classic
 
     LANGCHAIN_HUBS = {
         (
             RENAME_HUBS.get(hub.lower(), hub.lower())
             if hub.lower() in RENAME_HUBS
             else hub.lower()
         ): hub
-        for hub in langchain.llms.__all__
+        for hub in langchain_classic.llms.__all__
     }
     LANGCHAIN_HUBS["openrouter"] = "openrouter"
 

langtest/modelhandler/llm_modelhandler.py

@@ -3,9 +3,9 @@
 
 import os
 from typing import Any, List, Type, Union, TypeVar
-import langchain.llms as lc
+import langchain_classic.llms as lc
 import langchain.chat_models as chat_models
-from langchain.chains.llm import LLMChain
+from langchain_classic.chains.llm import LLMChain
 from langchain_core.prompts import PromptTemplate
 from langchain_core.language_models.base import BaseLanguageModel
 from langchain_core.exceptions import OutputParserException

langtest/modelhandler/modelhandler.py

@@ -14,16 +14,16 @@
 }
 
 if try_import_lib("langchain"):
-    import langchain
-    import langchain.llms
+    import langchain_classic
+    import langchain_classic.llms
 
     LANGCHAIN_HUBS = {
         (
             RENAME_HUBS.get(hub.lower(), hub.lower())
             if hub.lower() in RENAME_HUBS
             else hub.lower()
         ): hub
-        for hub in langchain.llms.__all__
+        for hub in langchain_classic.llms.__all__
     }
     LANGCHAIN_HUBS["openrouter"] = "openrouter"
 else: