Test updates

.github/copilot-instructions.md

@@ -0,0 +1,24 @@
+This is a Python based repository with a Python client for the Clarifai API. Please follow these
+guidelines when contributing:
+
+## Code Standards
+
+### Required Before Each Commit
+- Run `pre-commit run -a` before committing any changes to ensure proper code formatting and linting
+- This will run ruff on all Python files to maintain consistent style.
+- Fix any linter / formatting errors that are returned. This may require running multiple times.
+
+### Development Flow
+- Install dependencies: `(curl -LsSf https://astral.sh/uv/install.sh | sh) && uv venv && uv pip install -r requirements.txt -r tests/requirements.txt && pre-commit install`
+- Test: `./tests/run_tests.sh`
+
+## Repository Structure
+- `tests/`: Where all the tests are
+- `clarifai/`: The main Clarifai API client packages
+
+## Key Guidelines
+1. Follow Python best practices and idiomatic patterns
+2. Maintain existing code structure and organization
+3. Use dependency injection patterns where appropriate
+4. Write unit tests for new functionality. Use table-driven unit tests when possible.
+5. Document public APIs and complex logic. Suggest changes to the `README.md` folder when appropriate

.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,41 @@
+name: "Copilot Setup Steps"
+
+# Automatically run the setup steps when they are changed to allow for easy validation, and
+# allow manual testing through the repository's "Actions" tab
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+  pull_request:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+
+jobs:
+  # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+
+    # Set the permissions to the lowest permissions possible needed for your steps.
+    # Copilot will be given its own token for its operations.
+    permissions:
+      # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete.
+      contents: read
+
+    # You can define any steps you want, and they will run before the agent starts.
+    # If you do not check out your code, Copilot will do this for you.
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv
+          uv venv
+          uv pip install -r requirements.txt -r tests/requirements.txt

.github/workflows/run_tests.yml

@@ -13,10 +13,6 @@ on:
         description: "Boolean flag to diferentiate periodic checks"
         required: true
         type: string
-      CLARIFAI_ENV:
-        description: "Set the environment to run the tests"
-        required: true
-        type: string
       CLARIFAI_GRPC_BASE:
         description: "gRPC Base URL"
         required: true
@@ -67,11 +63,9 @@ jobs:
       shell: bash
       run: |
         if [ "${{ inputs.PERIODIC_CHECKS }}" = "true" ]; then
-          export CLARIFAI_ENV="${{ inputs.CLARIFAI_ENV }}"
           export CLARIFAI_GRPC_BASE="${{ inputs.CLARIFAI_GRPC_BASE }}"
           export CLARIFAI_API_BASE="https://${{ inputs.CLARIFAI_GRPC_BASE }}" # CLARIFAI_GRPC_BASE is used by the gRPC channel
         else
-          export CLARIFAI_ENV="prod"
           export CLARIFAI_GRPC_BASE="api.clarifai.com"
           export CLARIFAI_API_BASE="https://api.clarifai.com" # CLARIFAI_GRPC_BASE is used by the gRPC channel
         fi

.github/workflows/run_tests_prod.yml

@@ -10,6 +10,5 @@ jobs:
     uses: Clarifai/clarifai-python/.github/workflows/run_tests.yml@master
     with:
       PERIODIC_CHECKS: "true"
-      CLARIFAI_ENV: "prod"
       CLARIFAI_GRPC_BASE: "api.clarifai.com"
     secrets: inherit

scripts/key_for_tests.py

@@ -48,7 +48,15 @@ def _request(method, url, payload={}, headers={}):
 def login():
     url = "/login"
     payload = {"email": EMAIL, "password": PASSWORD}
-    data = _request(method="POST", url=url, payload=payload)
+    data = _request(
+        method="POST",
+        url=url,
+        payload=payload,
+        headers={
+            "Content-Type": "application/json",
+            "X-Clarifai-Client": "key_for_tests",
+        },
+    )
     _assert_response_success(data)
 
     assert "v2_user_id" in data, f"Invalid response {data}"
@@ -63,7 +71,11 @@ def login():
 
 
 def _auth_headers(session_token):
-    headers = {"Content-Type": "application/json", "X-Clarifai-Session-Token": session_token}
+    headers = {
+        "Content-Type": "application/json",
+        "X-Clarifai-Session-Token": session_token,
+        "X-Clarifai-Client": "key_for_tests",
+    }
     return headers
 
 

tests/cli/test_compute_orchestration.py

@@ -25,7 +25,6 @@
 DEPLOYMENT_CONFIG_FILE = "tests/compute_orchestration/configs/example_deployment_config.yaml"
 
 CLARIFAI_PAT = os.environ["CLARIFAI_PAT"]
-CLARIFAI_ENV = os.environ.get("CLARIFAI_ENV", "prod")
 CLARIFAI_API_BASE = os.environ.get("CLARIFAI_API_BASE", "https://api.clarifai.com")
 
 
@@ -109,7 +108,7 @@ def test_create_compute_cluster(self, cli_runner):
         with open(COMPUTE_CLUSTER_CONFIG_FILE, "w") as f:
             yaml.dump(config, f)
 
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(
             cli,
             [
@@ -129,7 +128,7 @@ def test_create_nodepool(self, cli_runner):
         with open(NODEPOOL_CONFIG_FILE, "w") as f:
             yaml.dump(config, f)
 
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(
             cli,
             [
@@ -153,7 +152,7 @@ def test_create_deployment(self, cli_runner):
         with open(DEPLOYMENT_CONFIG_FILE, "w") as f:
             yaml.dump(config, f)
 
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(
             cli,
             [
@@ -168,40 +167,40 @@ def test_create_deployment(self, cli_runner):
         assert result.exit_code == 0, logger.exception(result)
 
     def test_list_compute_clusters(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(cli, ["computecluster", "list"])
         assert result.exit_code == 0, logger.exception(result)
         assert "USER_ID" in result.output
 
     def test_list_nodepools(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(cli, ["nodepool", "list", CREATE_COMPUTE_CLUSTER_ID])
         assert result.exit_code == 0, logger.exception(result)
         assert "USER_ID" in result.output
 
     def test_list_deployments(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(cli, ["deployment", "list", CREATE_NODEPOOL_ID])
 
         assert result.exit_code == 0, logger.exception(result)
         assert "USER_ID" in result.output
 
     @pytest.mark.coverage_only
     def test_delete_deployment(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(
             cli, ["deployment", "delete", CREATE_NODEPOOL_ID, CREATE_DEPLOYMENT_ID]
         )
         assert result.exit_code == 0, logger.exception(result)
 
     def test_delete_nodepool(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(
             cli, ["nodepool", "delete", CREATE_COMPUTE_CLUSTER_ID, CREATE_NODEPOOL_ID]
         )
         assert result.exit_code == 0, logger.exception(result)
 
     def test_delete_compute_cluster(self, cli_runner):
-        cli_runner.invoke(cli, ["login", "--env", CLARIFAI_ENV])
+        cli_runner.invoke(cli, ["login"])
         result = cli_runner.invoke(cli, ["computecluster", "delete", CREATE_COMPUTE_CLUSTER_ID])
         assert result.exit_code == 0, logger.exception(result)

tests/compute_orchestration/configs/example_compute_cluster_config.yaml

@@ -1,10 +1,10 @@
 compute_cluster:
-  id: "test-aws-cluster"
-  description: "My AWS cluster"
   cloud_provider:
-      id: "aws"
-  region: "us-east-1"
-  managed_by: "clarifai"
-  cluster_type: "dedicated"
+    id: aws
+  cluster_type: dedicated
+  description: My AWS cluster
+  id: ci_test_cc_a4046969-a50c-40cd-b999-e28ec97d1405
+  managed_by: clarifai
+  region: us-east-1
   visibility:
     gettable: 10

tests/compute_orchestration/configs/example_deployment_config.yaml

@@ -1,23 +1,23 @@
 deployment:
-  id: "my_string_cat_8_thread_dep"
-  description: "some random deployment"
   autoscale_config:
-    min_replicas: 0
+    disable_packing: false
     max_replicas: 1
-    traffic_history_seconds: 100
+    min_replicas: 0
     scale_down_delay_seconds: 30
-    scale_up_delay_seconds: 30
     scale_to_zero_delay_seconds: 50
-    disable_packing: false
+    scale_up_delay_seconds: 30
+    traffic_history_seconds: 100
+  description: some random deployment
+  id: ci_test_dep_a4046969-a50c-40cd-b999-e28ec97d1405
+  nodepools:
+  - compute_cluster:
+      id: ci_test_cc_a4046969-a50c-40cd-b999-e28ec97d1405
+    id: ci_test_np_a4046969-a50c-40cd-b999-e28ec97d1405
+  scheduling_choice: 4
   worker:
     model:
-      id: "python_string_cat"
+      app_id: Test-Model-Upload
+      id: python_string_cat
       model_version:
-        id: "b7038e059a0c4ddca29c22aec561824d"
-      user_id: "clarifai"
-      app_id: "Test-Model-Upload"
-  scheduling_choice: 4
-  nodepools:
-    - id: "test-nodepool-6"
-      compute_cluster:
-          id: "test-aws-cluster"
+        id: b7038e059a0c4ddca29c22aec561824d
+      user_id: clarifai

tests/compute_orchestration/configs/example_nodepool_config.yaml

@@ -1,19 +1,19 @@
 nodepool:
-  id: "test-nodepool-6"
   compute_cluster:
-    id: "test-aws-cluster"
-  description: "First nodepool in AWS in a proper compute cluster"
+    id: test-aws-cluster
+  description: First nodepool in AWS in a proper compute cluster
+  id: ci_test_np_a4046969-a50c-40cd-b999-e28ec97d1405
   instance_types:
-    - id: "g5.xlarge"
-      compute_info:
-        cpu_limit: "8"
-        cpu_memory: "16Gi"
-        accelerator_type:
-          - "a10"
-        num_accelerators: 1
-        accelerator_memory: "40Gi"
-  node_capacity_type:
-    capacity_types:
-      - 1
+  - compute_info:
+      accelerator_memory: 40Gi
+      accelerator_type:
+      - a10
+      cpu_limit: '8'
+      cpu_memory: 16Gi
+      num_accelerators: 1
+    id: g5.xlarge
   max_instances: 1
   min_instances: 0
+  node_capacity_type:
+    capacity_types:
+    - 1

tests/requirements.txt

@@ -9,3 +9,4 @@ pypdf==3.17.4
 seaborn==0.13.2
 pycocotools>=2.0.7
 rich>=13.4.2
+pre-commit==4.2.0

tests/run_tests.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+# Before running make sure you run:
+# uv pip install -r requirements.txt -r tests/requirements.txt
+
+# You can now run this script to run the tests for the repo.
+# if you have your `clarifai config use {context}` set then you can run
+# eval $(clarifai config env) to source the variables into your shell and run_test.sh afterwards.
+# This allows you to hit different deployments of the Clarifai platform and using your own account.
+
+echo "Creating a new app and keys, so the test run is completely isolated from test runs."
+# if CLARIFAI_USER_ID does not exist, create it
+if [ -z "$CLARIFAI_USER_ID" ]; then
+    # you need to have CLARIFAI_USER_EMAIL and CLARIFAI_USER_PASSWORD set in your environment
+    if [ -z "$CLARIFAI_USER_EMAIL" ] || [ -z "$CLARIFAI_USER_PASSWORD" ]; then
+        echo "Please set CLARIFAI_USER_EMAIL and CLARIFAI_USER_PASSWORD environment variables."
+        exit 1
+    fi
+    export CLARIFAI_USER_ID="$(uv run python scripts/key_for_tests.py --get-userid)"
+fi
+# if CLARIFAI_PAT does not exist, create it
+if [ -z "$CLARIFAI_PAT" ]; then
+    # you need to have CLARIFAI_USER_EMAIL and CLARIFAI_USER_PASSWORD set in your environment
+    if [ -z "$CLARIFAI_USER_EMAIL" ] || [ -z "$CLARIFAI_USER_PASSWORD" ]; then
+        echo "Please set CLARIFAI_USER_EMAIL and CLARIFAI_USER_PASSWORD environment variables."
+        exit 1
+    fi
+    export CLARIFAI_PAT="$(uv run python scripts/key_for_tests.py --create-pat)"
+fi
+# if CLARIFAI_API_BASE is not set, default to https://api.clarifai.com
+if [ -z "$CLARIFAI_API_BASE" ]; then
+    export CLARIFAI_API_BASE="https://api.clarifai.com"
+fi
+# if CLARIFAI_GRPC_BASE is not set, default to api.clarifai.com
+if [ -z "$CLARIFAI_GRPC_BASE" ]; then
+    export CLARIFAI_GRPC_BASE="api.clarifai.com"
+fi
+
+# First run the linter tests to make sure those pass.
+uv run pre-commit run --all-files
+
+
+echo "Running all the tests"
+# See .github/workflows/run_tests.yml as there are more combinations of tests that can be run but
+# this should cover the basics.
+uv run pytest -s tests/ -v -n auto --durations=5 --timeout=1800 --ignore=tests/runners/test_model_run_locally-container.py
+
+
+# TODO: cleanup better from these tests capturing the exit code first.
+# test_result=$?
+# echo "Running single test"
+# python3 -m pytest tests/ -vvv -s -k "test_predict_image_url_with_min_value"
+# echo "Deleting the created application"
+# python3 scripts/app_and_key_for_tests.py --delete-app ${CLARIFAI_APP_ID}
+# exit $test_result