Separate manual tests and remove unused settings from E2E (#874)

* Separate manual tests and remove unused settings from E2E

* Remove non sn/i test case

* Remove unused test

* Improve ADO logs

* Skip some tests on ADO

Author: bgavrilMS

.github/workflows/python-package.yml

@@ -17,12 +17,8 @@ jobs:
       # Fake a TRAVIS env so that the pre-existing test cases would behave like before
       TRAVIS: true
       LAB_APP_CLIENT_ID: ${{ secrets.LAB_APP_CLIENT_ID }}
-      LAB_APP_CLIENT_SECRET: ${{ secrets.LAB_APP_CLIENT_SECRET }}
       LAB_APP_CLIENT_CERT_BASE64: ${{ secrets.LAB_APP_CLIENT_CERT_BASE64 }}
       LAB_APP_CLIENT_CERT_PFX_PATH: lab_cert.pfx
-      LAB_OBO_CLIENT_SECRET: ${{ secrets.LAB_OBO_CLIENT_SECRET }}
-      LAB_OBO_CONFIDENTIAL_CLIENT_ID: ${{ secrets.LAB_OBO_CONFIDENTIAL_CLIENT_ID }}
-      LAB_OBO_PUBLIC_CLIENT_ID: ${{ secrets.LAB_OBO_PUBLIC_CLIENT_ID }}
 
     # Derived from https://docs.github.com/en/actions/guides/building-and-testing-python#starting-with-the-python-workflow-template
     runs-on: ubuntu-22.04

azure-pipelines.yml

@@ -37,5 +37,23 @@ steps:
 
 - script: |
     pip install pytest pytest-azurepipelines
-    pytest
-  displayName: 'pytest'
+    mkdir -p test-results
+    set -o pipefail
+    pytest -vv --junitxml=test-results/junit.xml 2>&1 | tee test-results/pytest.log
+  displayName: 'pytest (verbose + junit + log)'
+
+- task: PublishTestResults@2
+  displayName: 'Publish test results'
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFormat: 'JUnit'
+    testResultsFiles: 'test-results/junit.xml'
+    failTaskOnFailedTests: true
+    testRunTitle: 'Python $(python.version) pytest'
+
+- task: PublishPipelineArtifact@1
+  displayName: 'Publish pytest log artifact'
+  condition: succeededOrFailed()
+  inputs:
+    targetPath: 'test-results'
+    artifact: 'pytest-logs-$(python.version)'

tests/README.md

@@ -0,0 +1,87 @@
+# Local test setup
+
+This document explains how to set up a local development environment to run tests in this repo, including E2E tests.
+
+## 1) Prerequisites
+
+- Windows, macOS, or Linux
+- Python 3.9+
+- Access to the MSAL lab secrets (Key Vault) for E2E tests
+- A registered lab app credential: i.e. certificate `.pfx` file path
+
+## 2) Create and activate a virtual environment
+
+From repo root:
+
+```powershell
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1
+python -m pip install --upgrade pip
+python -m pip install -r requirements.txt
+```
+
+## 3) Configure environment variables
+
+Create a local `.env` file in repo root (same folder as `setup.py`):
+
+```dotenv
+LAB_APP_CLIENT_ID=<your-lab-app-client-id>
+LAB_APP_CLIENT_CERT_PFX_PATH=C:/path/to/your/cert.pfx
+
+```
+
+Notes:
+- `tests/test_e2e.py` loads `.env` automatically when `python-dotenv` is installed.
+- For certificate auth, `LAB_APP_CLIENT_CERT_PFX_PATH` should be an absolute path.
+
+## 4) Run unit/integration tests
+
+Run all non-E2E tests quickly:
+
+```powershell
+python -m pytest -q tests -k "not e2e"
+```
+
+Run full E2E unattended suite:
+
+```powershell
+python -m pytest -q tests/test_e2e.py
+```
+
+## 5) Manual-intervention E2E tests
+
+Manual tests (interactive browser/device-flow/POP manual scenarios) are separated into:
+
+- `tests/test_e2e_manual.py`
+
+By default they are skipped. To enable:
+
+```powershell
+$env:RUN_MANUAL_E2E = "1"
+python -m pytest -q tests/test_e2e_manual.py
+```
+
+To disable again in the current shell:
+
+```powershell
+Remove-Item Env:RUN_MANUAL_E2E
+```
+
+## 6) Common troubleshooting
+
+### AADSTS700027 / invalid_client for certificate flow
+
+If you see errors indicating SNI/x5c is required, your app registration may only accept certificate auth with x5c chain. In this repo, that path is covered by SNI-oriented cert tests.
+
+### Key Vault access failures
+
+Verify:
+- `LAB_APP_CLIENT_ID` is correct
+- `LAB_APP_CLIENT_CERT_PFX_PATH` points to a valid `.pfx` file
+- Your principal has access to:
+  - `https://msidlabs.vault.azure.net`
+  - `https://id4skeyvault.vault.azure.net`
+
+### Interactive tests unexpectedly skipped
+
+Interactive/manual tests are intentionally gated. Set `RUN_MANUAL_E2E=1` and run `tests/test_e2e_manual.py`.

tests/lab_config.py

@@ -21,8 +21,7 @@
 
 Environment Variables:
     LAB_APP_CLIENT_ID: Client ID for Key Vault authentication (required)
-    LAB_APP_CLIENT_CERT_PFX_PATH: Path to .pfx certificate file (preferred)
-    LAB_APP_CLIENT_SECRET: Client secret (alternative to certificate)
+    LAB_APP_CLIENT_CERT_PFX_PATH: Path to .pfx certificate file (required)
 """
 
 import json
@@ -31,7 +30,7 @@
 from dataclasses import dataclass
 from typing import Dict, Optional
 
-from azure.identity import CertificateCredential, ClientSecretCredential
+from azure.identity import CertificateCredential
 from azure.keyvault.secrets import SecretClient
 
 logger = logging.getLogger(__name__)
@@ -169,9 +168,8 @@ def _get_credential():
     """
     Create an Azure credential for Key Vault access.
     
-    Reads authentication details from environment variables. Prefers
-    certificate-based authentication if LAB_APP_CLIENT_CERT_PFX_PATH is set,
-    otherwise falls back to client secret.
+    Reads authentication details from environment variables and uses
+    certificate-based authentication via LAB_APP_CLIENT_CERT_PFX_PATH.
     
     Returns:
         A credential object suitable for Azure SDK clients.
@@ -180,7 +178,6 @@ def _get_credential():
         EnvironmentError: If required environment variables are not set.
     """
     client_id = os.getenv("LAB_APP_CLIENT_ID")
-    client_secret = os.getenv("LAB_APP_CLIENT_SECRET")
     cert_path = os.getenv("LAB_APP_CLIENT_CERT_PFX_PATH")
     tenant_id = "72f988bf-86f1-41af-91ab-2d7cd011db47"  # Microsoft tenant
 
@@ -196,16 +193,9 @@ def _get_credential():
             certificate_path=cert_path,
             send_certificate_chain=True,
         )
-    elif client_secret:
-        logger.debug("Using client secret credential for Key Vault access")
-        return ClientSecretCredential(
-            tenant_id=tenant_id,
-            client_id=client_id,
-            client_secret=client_secret,
-        )
     else:
         raise EnvironmentError(
-            "Either LAB_APP_CLIENT_SECRET or LAB_APP_CLIENT_CERT_PFX_PATH is required")
+            "LAB_APP_CLIENT_CERT_PFX_PATH is required")
 
 
 def _get_msid_lab_client() -> SecretClient: