CLOUD-929 - Initial pytest test migration

.github/workflows/e2e-py-check.yml

@@ -0,0 +1,34 @@
+name: e2e-tests Python Quality Check
+
+on:
+  pull_request:
+    paths:
+      - 'e2e-tests/**/*.py'
+
+permissions:
+  contents: read
+
+jobs:
+  quality-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync --locked
+
+      - name: Run ruff check
+        run: uv run ruff check e2e-tests/
+
+      - name: Run mypy
+        run: uv run mypy e2e-tests/

.gitignore

@@ -189,3 +189,7 @@ bin/
 projects/
 installers/olm/operator_*.yaml
 installers/olm/bundles
+
+# Test Reports
+e2e-tests/reports/
+e2e-tests/**/__pycache__/

Jenkinsfile

@@ -103,6 +103,16 @@ void pushLogFile(String FILE_NAME) {
     }
 }
 
+void pushReportFile() {
+    echo "Push final_report.html to S3!"
+    withCredentials([aws(credentialsId: 'AMI/OVF', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY')]) {
+        sh """
+            S3_PATH=s3://percona-jenkins-artifactory-public/\$JOB_NAME/\$(git rev-parse --short HEAD)
+            aws s3 cp --content-type text/html --quiet final_report.html \$S3_PATH/final_report.html || :
+        """
+    }
+}
+
 void pushArtifactFile(String FILE_NAME) {
     echo "Push $FILE_NAME file to S3!"
 
@@ -148,24 +158,6 @@ void markPassedTests() {
     }
 }
 
-void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
-    sh """
-        export KUBECONFIG=/tmp/${CLUSTER_NAME}-${CLUSTER_SUFFIX}
-        echo "========== KUBERNETES STATUS $LOCATION TEST =========="
-        gcloud container clusters list|grep -E "NAME|${CLUSTER_NAME}-${CLUSTER_SUFFIX} "
-        echo
-        kubectl get nodes
-        echo
-        kubectl top nodes
-        echo
-        kubectl get pods --all-namespaces
-        echo
-        kubectl top pod --all-namespaces
-        echo
-        kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp"
-        echo "======================================================"
-    """
-}
 
 String formatTime(def time) {
     if (!time || time == "N/A") return "N/A"
@@ -185,7 +177,6 @@ String formatTime(def time) {
 }
 
 @Field def TestsReport = '| Test Name | Result | Time |\r\n| ----------- | -------- | ------ |'
-@Field def TestsReportXML = '<testsuite name=\\"PSMDB\\">\n'
 
 void makeReport() {
     def wholeTestAmount = tests.size()
@@ -206,17 +197,62 @@ void makeReport() {
             startedTestAmount++
         }
         TestsReport = TestsReport + "\r\n| " + testName + " | [" + testResult + "](" + testUrl + ") | " + formatTime(testTime) + " |"
-        TestsReportXML = TestsReportXML + '<testcase name=\\"' + testName + '\\" time=\\"' + testTime + '\\"><'+ testResult +'/></testcase>\n'
     }
     TestsReport = TestsReport + "\r\n\r\n| Summary | Value |\r\n| ------- | ----- |"
     TestsReport = TestsReport + "\r\n| Tests Run | $startedTestAmount/$wholeTestAmount |"
     TestsReport = TestsReport + "\r\n| Job Duration | " + formatTime(currentBuild.duration / 1000) + " |"
     TestsReport = TestsReport + "\r\n| Total Test Time | "  + formatTime(totalTestTime) + " |"
-    TestsReportXML = TestsReportXML + '</testsuite>\n'
+}
 
-    sh """
-        echo "${TestsReportXML}" > TestsReport.xml
-    """
+void generateMissingReports() {
+    sh "mkdir -p e2e-tests/reports"
+
+    for (int i = 0; i < tests.size(); i++) {
+        def testName = tests[i]["name"]
+        def testResult = tests[i]["result"]
+        def testTime = tests[i]["time"] ?: 0
+
+        if (testResult == "skipped") {
+            continue
+        }
+
+        def xmlFile = "e2e-tests/reports/${testName}.xml"
+        def htmlFile = "e2e-tests/reports/${testName}.html"
+
+        if (!fileExists(xmlFile)) {
+            def failures = testResult == "failure" ? 1 : 0
+            def failureElement = testResult == "failure" ?
+                '<failure message="Missing report">Test did not produce a report</failure>' : ''
+
+            writeFile file: xmlFile, text: """<?xml version="1.0" encoding="utf-8"?>
+<testsuites name="pytest tests">
+<testsuite name="psmdb-e2e" errors="0" failures="${failures}" skipped="0" tests="1" time="${testTime}">
+<testcase classname="e2e-tests.${testName}" name="${testName}" time="${testTime}">
+${failureElement}
+</testcase>
+</testsuite>
+</testsuites>"""
+        }
+
+        if (!fileExists(htmlFile)) {
+            def resultCapitalized = testResult == "failure" ? "Failed" : "Passed"
+            def formattedTime = formatTime(testTime)
+            def logMessage = testResult == "failure" ?
+                "Test did not produce a report" :
+                "Test marked as passed (from previous run)"
+
+            writeFile file: htmlFile, text: """<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8"/>
+<title id="head-title">${testName}.html</title>
+</head>
+<body>
+<div id="data-container" data-jsonblob='{"environment": {"Note": "Placeholder report generated because the test report was missing"}, "tests": {"${testName}": [{"extras": [], "result": "${resultCapitalized}", "testId": "${testName}", "duration": "${formattedTime}", "resultsTableRow": ["<td class=\\"col-result\\">${resultCapitalized}</td>", "<td>-</td>", "<td class=\\"col-testId\\">${testName}</td>", "<td class=\\"col-duration\\">${formattedTime}</td>", "<td class=\\"col-links\\"></td>"], "log": "${logMessage}"}]}}'></div>
+</body>
+</html>"""
+        }
+    }
 }
 
 void clusterRunner(String cluster) {
@@ -242,48 +278,48 @@ void clusterRunner(String cluster) {
 }
 
 void runTest(Integer TEST_ID) {
-    def retryCount = 0
     def testName = tests[TEST_ID]["name"]
     def clusterSuffix = tests[TEST_ID]["cluster"]
+    def timeStart = new Date().getTime()
 
-    waitUntil {
-        def timeStart = new Date().getTime()
-        try {
-            echo "The $testName test was started on cluster ${CLUSTER_NAME}-${clusterSuffix} !"
-            tests[TEST_ID]["result"] = "failure"
-
-            timeout(time: 90, unit: 'MINUTES') {
-                sh """
-                    if [ $retryCount -eq 0 ]; then
-                        export DEBUG_TESTS=0
-                    else
-                        export DEBUG_TESTS=1
-                    fi
-                    export KUBECONFIG=/tmp/${CLUSTER_NAME}-${clusterSuffix}
-                    time ./e2e-tests/$testName/run
-                """
-            }
-            pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testName")
-            tests[TEST_ID]["result"] = "passed"
-            return true
-        }
-        catch (exc) {
-            printKubernetesStatus("AFTER","$clusterSuffix")
-            echo "Test $testName has failed!"
-            if (retryCount >= 1 || currentBuild.nextBuild != null) {
-                currentBuild.result = 'FAILURE'
-                return true
-            }
-            retryCount++
-            return false
-        }
-        finally {
-            def timeStop = new Date().getTime()
-            def durationSec = (timeStop - timeStart) / 1000
-            tests[TEST_ID]["time"] = durationSec
-            pushLogFile("$testName")
-            echo "The $testName test was finished!"
+    try {
+        echo "The $testName test was started on cluster ${CLUSTER_NAME}-${clusterSuffix} !"
+        tests[TEST_ID]["result"] = "failure"
+
+        timeout(time: 90, unit: 'MINUTES') {
+            sh """
+                export DEBUG_TESTS=1
+                export KUBECONFIG=/tmp/${CLUSTER_NAME}-${clusterSuffix}
+                export PATH="\$HOME/.local/bin:\$PATH"
+                mkdir -p e2e-tests/reports
+
+                REPORT_OPTS="--html=e2e-tests/reports/${testName}.html --junitxml=e2e-tests/reports/${testName}.xml"
+
+                # Run native pytest if test_*.py exists, otherwise run bash via wrapper
+                if ls e2e-tests/$testName/test_*.py 1>/dev/null 2>&1; then
+                    uv run pytest e2e-tests/$testName/ \$REPORT_OPTS
+                else
+                    uv run pytest e2e-tests/test_pytest_wrapper.py --test-name=$testName \$REPORT_OPTS
+                fi
+            """
         }
+        pushArtifactFile("${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}-$testName")
+        tests[TEST_ID]["result"] = "passed"
+    }
+    catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException exc) {
+        echo "Test $testName was interrupted!"
+        throw exc
+    }
+    catch (exc) {
+        echo "Test $testName has failed!"
+        currentBuild.result = 'FAILURE'
+    }
+    finally {
+        def timeStop = new Date().getTime()
+        def durationSec = (timeStop - timeStart) / 1000
+        tests[TEST_ID]["time"] = durationSec
+        pushLogFile("$testName")
+        echo "The $testName test was finished!"
     }
 }
 
@@ -309,6 +345,10 @@ EOF
         sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
 
         curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
+
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        export PATH="\$HOME/.local/bin:\$PATH"
+        uv sync --locked
     """
     installAzureCLI()
     azureAuth()
@@ -431,7 +471,7 @@ pipeline {
         GIT_SHORT_COMMIT = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
         VERSION = "${env.GIT_BRANCH}-${env.GIT_SHORT_COMMIT}"
         CLUSTER_NAME = sh(script: "echo jen-psmdb-${env.CHANGE_ID}-${GIT_SHORT_COMMIT}-${env.BUILD_NUMBER} | tr '[:upper:]' '[:lower:]'", returnStdout: true).trim()
-        AUTHOR_NAME = sh(script: "echo ${CHANGE_AUTHOR_EMAIL} | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
+        AUTHOR_NAME = sh(script: "echo ${CHANGE_AUTHOR_EMAIL} | awk -F'@' '{print \$1}'", returnStdout: true).trim()
         ENABLE_LOGGING = "true"
     }
     agent {
@@ -463,7 +503,7 @@ pipeline {
                 prepareNode()
                 script {
                     if (AUTHOR_NAME == 'null') {
-                        AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
+                        AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", returnStdout: true).trim()
                     }
                     for (comment in pullRequest.comments) {
                         println("Author: ${comment.user}, Comment: ${comment.body}")
@@ -680,12 +720,20 @@ pipeline {
                             }
                         }
                         makeReport()
-                        junit testResults: '*.xml', healthScaleFactor: 1.0
-                        archiveArtifacts '*.xml'
+                        generateMissingReports()
+
+                        sh """
+                            export PATH="\$HOME/.local/bin:\$PATH"
+                            uv run pytest_html_merger -i e2e-tests/reports -o final_report.html
+                            uv run junitparser merge --glob 'e2e-tests/reports/*.xml' final_report.xml
+                        """
+                        junit testResults: 'final_report.xml', healthScaleFactor: 1.0
+                        archiveArtifacts 'final_report.xml, final_report.html'
+                        pushReportFile()
 
                         unstash 'IMAGE'
                         def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
-                        TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
+                        TestsReport = TestsReport + "\r\n\r\nCommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nImage: `${IMAGE}`\r\nTest report: [report](${testUrlPrefix}/${env.GIT_BRANCH}/${env.GIT_SHORT_COMMIT}/final_report.html)\r\n"
                         pullRequest.comment(TestsReport)
                     }
                     deleteOldClusters("$CLUSTER_NAME")

Makefile

@@ -71,6 +71,20 @@ undeploy: ## Undeploy operator
 test: envtest generate ## Run tests.
 	DISABLE_TELEMETRY=true KUBEBUILDER_ASSETS="$(shell $(ENVTEST) --arch=amd64 use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
 
+py-deps: uv ## Install e2e-tests Python dependencies
+	$(UV) sync --locked
+
+py-update-deps: uv ## Update e2e-tests Python dependencies
+	$(UV) lock --upgrade
+
+py-fmt: uv ## Format and organize imports in e2e-tests
+	$(UV) run ruff check --select I --fix e2e-tests/
+	$(UV) run ruff format e2e-tests/
+
+py-check: uv ## Run ruff and mypy checks on e2e-tests
+	$(UV) run ruff check e2e-tests/
+	$(UV) run mypy e2e-tests/
+
 # go-get-tool will 'go get' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 define go-get-tool
@@ -105,6 +119,12 @@ MOCKGEN = $(shell pwd)/bin/mockgen
 mockgen: ## Download mockgen locally if necessary.
 	$(call go-get-tool,$(MOCKGEN), github.com/golang/mock/mockgen@latest)
 
+UV = $(shell pwd)/bin/uv
+uv: ## Download uv locally if necessary.
+	@[ -f $(UV) ] || { \
+	set -e ;\
+	curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=$(PROJECT_DIR)/bin sh ;\
+	}
 update-version:
 	echo $(NEXT_VER) > pkg/version/version.txt