chore: enhance GitHub Actions workflow and test cases

MervinPraison · MervinPraison · commit 4a65f7760e89 · 2025-05-28T17:58:44.000+01:00
- Added debugging steps to check API key availability and log environment variables in the GitHub Actions workflow.
- Updated test cases to remove API key validation checks, ensuring all tests run regardless of API key status.
- Renamed test steps for clarity and improved readability.

These changes improve the testing process and provide better insights during execution.
diff --git a/.github/workflows/test-core.yml b/.github/workflows/test-core.yml
@@ -40,8 +40,64 @@ jobs:
         echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY || 'sk-test-key-for-github-actions-testing-only-not-real' }}" >> $GITHUB_ENV
         echo "OPENAI_API_BASE=${{ secrets.OPENAI_API_BASE || 'https://api.openai.com/v1' }}" >> $GITHUB_ENV
         echo "OPENAI_MODEL_NAME=${{ secrets.OPENAI_MODEL_NAME || 'gpt-4o-mini' }}" >> $GITHUB_ENV
+        echo "LOGLEVEL=DEBUG" >> $GITHUB_ENV
         echo "PYTHONPATH=${{ github.workspace }}/src/praisonai-agents:$PYTHONPATH" >> $GITHUB_ENV
 
+    - name: Debug API Key Status
+      run: |
+        echo "🔍 Checking API key availability..."
+        if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then
+          echo "✅ GitHub secret OPENAI_API_KEY is available"
+          echo "🔑 API key starts with: $(echo "$OPENAI_API_KEY" | cut -c1-7)..."
+        else
+          echo "⚠️ GitHub secret OPENAI_API_KEY is NOT set - using fallback"
+          echo "🔑 Using fallback key: sk-test-key..."
+        fi
+        echo "🌐 API Base: $OPENAI_API_BASE"
+        echo "🤖 Model: $OPENAI_MODEL_NAME"
+        echo "🐛 Log Level: $LOGLEVEL"
+        echo "📊 Environment Check:"
+        echo "  - OPENAI_API_KEY length: ${#OPENAI_API_KEY}"
+        echo "  - OPENAI_API_BASE: $OPENAI_API_BASE"
+        echo "  - OPENAI_MODEL_NAME: $OPENAI_MODEL_NAME"
+        echo "  - LOGLEVEL: $LOGLEVEL"
+
+    - name: Debug Python Environment Variables  
+      run: |
+        python -c "
+        import os
+        print('🐍 Python Environment Variable Check:')
+        api_key = os.environ.get('OPENAI_API_KEY', 'NOT_SET')
+        if api_key != 'NOT_SET':
+            print(f'  ✅ OPENAI_API_KEY: {api_key[:7]}... (length: {len(api_key)})')
+        else:
+            print('  ❌ OPENAI_API_KEY: NOT_SET')
+        print(f'  🌐 OPENAI_API_BASE: {os.environ.get(\"OPENAI_API_BASE\", \"NOT_SET\")}')
+        print(f'  🤖 OPENAI_MODEL_NAME: {os.environ.get(\"OPENAI_MODEL_NAME\", \"NOT_SET\")}')
+        print(f'  📋 All OPENAI env vars:')
+        for key, value in os.environ.items():
+            if key.startswith('OPENAI'):
+                print(f'    {key}: {value[:10] if len(value) > 10 else value}...')
+        "
+
+    - name: Validate API Key
+      run: |
+        echo "🔑 Testing API key validity with minimal OpenAI call..."
+        python -c "
+        import os
+        try:
+            from openai import OpenAI
+            client = OpenAI(api_key=os.environ.get('OPENAI_API_KEY'))
+            response = client.models.list()
+            print('✅ API Key is VALID - OpenAI responded successfully')
+            print(f'📊 Available models: {len(list(response.data))} models found')
+        except Exception as e:
+            print(f'❌ API Key is INVALID - Error: {e}')
+            print('🔍 This explains why all API-dependent tests are failing')
+            print('💡 The GitHub secret OPENAI_API_KEY needs to be updated with a valid key')
+        "
+      continue-on-error: true
+
     - name: Run Unit Tests
       run: |
         cd src/praisonai && python -m pytest tests/unit/ -v --tb=short --disable-warnings --cov=praisonai --cov-report=term-missing
@@ -50,23 +106,20 @@ jobs:
       run: |
         cd src/praisonai && python -m pytest tests/integration/ -v --tb=short --disable-warnings
 
-    - name: Run AutoGen Framework Tests
+    - name: Test AutoGen Framework
       run: |
         echo "🤖 Testing AutoGen Framework Integration..."
-        cd src/praisonai && python tests/test_runner.py --pattern autogen --verbose || echo "⚠️ AutoGen tests completed with issues"
-      continue-on-error: true
+        cd src/praisonai && python tests/test_runner.py --pattern autogen --verbose
 
-    - name: Run CrewAI Framework Tests  
+    - name: Test CrewAI Framework
       run: |
         echo "⛵ Testing CrewAI Framework Integration..."
-        cd src/praisonai && python tests/test_runner.py --pattern crewai --verbose || echo "⚠️ CrewAI tests completed with issues"
-      continue-on-error: true
+        cd src/praisonai && python tests/test_runner.py --pattern crewai --verbose
 
-    - name: Run Legacy Tests (with skip on invalid API key)
+    - name: Run Legacy Tests with API Key
       run: |
-        echo "🧪 Running legacy tests - these should skip if API key is invalid..."
-        cd src/praisonai && OPENAI_API_KEY="sk-test-key-for-github-actions-testing-only-not-real" python -m pytest tests/test.py -v --tb=short --disable-warnings
-      continue-on-error: true
+        echo "🧪 Running legacy tests with real API key..."
+        cd src/praisonai && python -m pytest tests/test.py -v --tb=short --disable-warnings
 
     - name: Upload Coverage Reports
       if: matrix.python-version == '3.11'
diff --git a/src/praisonai/tests/test.py b/src/praisonai/tests/test.py
@@ -11,153 +11,74 @@
 import collections.abc
 collections.MutableMapping = collections.abc.MutableMapping
 
-def _should_skip_api_test():
-    """Check if API tests should be skipped due to invalid/test API key"""
-    api_key = os.environ.get('OPENAI_API_KEY', '')
-    return (not api_key or 
-            api_key.startswith('sk-test-') or 
-            api_key == 'nokey' or
-            api_key == 'test-key' or  # Handle truncated test key
-            'fallback' in api_key or
-            'testing-only' in api_key or
-            'not-real' in api_key)
-
 class TestPraisonAIFramework(unittest.TestCase):
     def test_main_with_agents_advanced(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         praisonai = PraisonAI(agent_file="tests/agents-advanced.yaml")
         result = praisonai.run()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
 
     def test_main_with_autogen_framework(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         praisonai = PraisonAI(agent_file="tests/autogen-agents.yaml")
         result = praisonai.run()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
 
     def test_main_with_custom_framework(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         praisonai = PraisonAI(agent_file="tests/crewai-agents.yaml")
         result = praisonai.run()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
 
     def test_main_with_internet_search_tool(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         praisonai = PraisonAI(agent_file="tests/search-tool-agents.yaml")
         result = praisonai.run()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
 
     def test_main_with_built_in_tool(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
-        praisonai = PraisonAI(agent_file="tests/inbuilt-tool-agents.yaml")
+        praisonai = PraisonAI(agent_file="tests/built-in-tools-agents.yaml")
         result = praisonai.run()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
     
 
 class TestPraisonAICommandLine(unittest.TestCase):
     def run_command(self, command):
-        result = subprocess.run(
-            command, 
-            stdout=subprocess.PIPE, 
-            stderr=subprocess.STDOUT, 
-            shell=True, 
-            text=True, 
-            env=os.environ.copy()  # Fix: Inherit environment variables
-        )
-        return result.stdout
+        """Helper method to run CLI commands"""
+        result = subprocess.run(command, shell=True, capture_output=True, text=True)
+        return result.stdout + result.stderr
 
     def test_praisonai_command(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         # Test basic praisonai command
         command = "praisonai --framework autogen --auto create a 2-agent team to write a simple python game"
         result = self.run_command(command)
         print(f"Result: {result}")
         self.assertIn('TERMINATE', result)
 
     def test_praisonai_init_command(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         # Test praisonai --init command
         command = "praisonai --framework autogen --init create a 2-agent team to write a simple python game"
         result = self.run_command(command)
         print(f"Result: {result}")
         self.assertIn('created successfully', result)
 
 class TestExamples(unittest.TestCase):
-    def test_basic_example(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping due to no valid API key provided")
-            
-        # Test the basic example function
-        try:
-            result = main()
-            self.assertIsNotNone(result)
-            # Check if result contains expected success indicators or output
-            self.assertTrue(
-                isinstance(result, str) and (
-                    "completed successfully" in result or 
-                    "Task Output" in result or
-                    len(result.strip()) > 0
-                ),
-                f"Expected meaningful result, got: {result}"
-            )
-        except Exception as e:
-            # Fallback skip for API errors
-            if any(error in str(e) for error in ['Invalid API Key', 'AuthenticationError', '401']):
-                self.skipTest(f"Skipping due to API key issue: {e}")
-            else:
-                raise
-
     def test_advanced_example(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping due to no valid API key provided")
-            
-        # Test the advanced example function
-        try:
-            result = advanced()
-            self.assertIsNotNone(result)
-            # Check if result contains expected success indicators or output
-            self.assertTrue(
-                isinstance(result, str) and (
-                    "completed successfully" in result or 
-                    "Task Output" in result or
-                    len(result.strip()) > 0
-                ),
-                f"Expected meaningful result, got: {result}"
-            )
-        except Exception as e:
-            # Fallback skip for API errors
-            if any(error in str(e) for error in ['Invalid API Key', 'AuthenticationError', '401']):
-                self.skipTest(f"Skipping due to API key issue: {e}")
-            else:
-                raise
+        result = advanced()
+        print(f"Result: {result}")
+        self.assertIsNotNone(result)
 
     def test_auto_example(self):
-        if _should_skip_api_test():
-            self.skipTest("Skipping API test due to invalid/test API key")
-        
         result = auto()
         print(f"Result: {result}")
         self.assertIsNotNone(result)
 
+    def test_basic_example(self):
+        result = main()
+        print(f"Result: {result}")
+        self.assertIsNotNone(result)
+
 if __name__ == '__main__':
     # runner = XMLTestRunner(output='test-reports')
     unittest.main()
