tests: add new marker and new github action for 3 times per day gpu intensive tests

Author: lucasjia-aws

.github/workflows/gpu-integ-tests.yml

@@ -0,0 +1,24 @@
+name: GPU Integ Tests
+on:
+  schedule:
+    - cron: "0 */8 * * *"
+  workflow_dispatch:
+
+permissions:
+    id-token: write # This is required for requesting the JWT
+
+jobs:
+  gpu-integ-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 10800
+      - name: Run GPU Integ Tests
+        uses: aws-actions/aws-codebuild-run-build@v1
+        with:
+          project-name: sagemaker-python-sdk-ci-health-gpu-integ-tests
+          source-version: refs/heads/master

sagemaker-train/tests/integ/train/test_benchmark_evaluator.py

@@ -23,6 +23,8 @@
     EvaluationPipelineExecution,
 )
 
+pytestmark = pytest.mark.gpu_intensive
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,

sagemaker-train/tests/integ/train/test_custom_scorer_evaluator.py

@@ -22,6 +22,8 @@
     EvaluationPipelineExecution,
 )
 
+pytestmark = pytest.mark.gpu_intensive
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,

sagemaker-train/tests/integ/train/test_dpo_trainer_integration.py

@@ -21,6 +21,8 @@
 from sagemaker.train.common import TrainingType
 import pytest
 
+pytestmark = pytest.mark.gpu_intensive
+
 
 def test_dpo_trainer_lora_complete_workflow(sagemaker_session):
     """Test complete DPO training workflow with LORA."""

sagemaker-train/tests/integ/train/test_llm_as_judge_base_model_fix.py

@@ -28,6 +28,8 @@
     EvaluationPipelineExecution,
 )
 
+pytestmark = pytest.mark.gpu_intensive
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,

sagemaker-train/tests/integ/train/test_llm_as_judge_evaluator.py

@@ -22,6 +22,8 @@
     EvaluationPipelineExecution,
 )
 
+pytestmark = pytest.mark.gpu_intensive
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,

sagemaker-train/tests/integ/train/test_rlaif_trainer_integration.py

@@ -21,6 +21,8 @@
 from sagemaker.train.common import TrainingType
 import pytest
 
+pytestmark = pytest.mark.gpu_intensive
+
 
 def test_rlaif_trainer_lora_complete_workflow(sagemaker_session):
     """Test complete RLAIF training workflow with LORA."""

sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py

@@ -21,6 +21,8 @@
 from sagemaker.train.rlvr_trainer import RLVRTrainer
 from sagemaker.train.common import TrainingType
 
+pytestmark = pytest.mark.gpu_intensive
+
 
 def test_rlvr_trainer_lora_complete_workflow(sagemaker_session):
     """Test complete RLVR training workflow with LORA."""

sagemaker-train/tests/integ/train/test_sft_trainer_integration.py

@@ -21,6 +21,8 @@
 from sagemaker.train.sft_trainer import SFTTrainer
 from sagemaker.train.common import TrainingType
 
+pytestmark = pytest.mark.gpu_intensive
+
 
 def test_sft_trainer_lora_complete_workflow(sagemaker_session):
     """Test complete SFT training workflow with LORA."""

sagemaker-train/tests/integ/train/test_tuner_distributed.py

@@ -28,6 +28,8 @@
 from sagemaker.train.configs import SourceCode, Compute
 from sagemaker.train.distributed import Torchrun
 from sagemaker.train.tuner import HyperparameterTuner
+
+pytestmark = pytest.mark.gpu_intensive
 from sagemaker.core.parameter import ContinuousParameter
 
 logger = logging.getLogger(__name__)