Merge pull request #651 from callstack-internal/bugfix/reassure-flakiness-v2

[NoQA] Investigate Reassure flakiness v2

Author: stitesExpensify

.github/actions/javascript/validateReassureOutput/action.yml

@@ -7,6 +7,9 @@ inputs:
     ALLOWED_RELATIVE_DURATION_DEVIATION:
         description: Allowable percentage deviation for the mean duration in regression test results.
         required: true
+    IS_VALIDATING_STABILITY:
+        description: Whether the workflow is validating a Reassure stability check or not.
+        required: true
 runs:
     using: 'node20'
     main: './index.js'

.github/actions/javascript/validateReassureOutput/index.js

@@ -24972,6 +24972,7 @@ function run() {
             const regressionOutput = JSON.parse(fs_1.default.readFileSync('.reassure/output.json', 'utf8'));
             const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', { required: true }));
             const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', { required: true }));
+            const isValidatingStability = Boolean(core.getInput('IS_VALIDATING_STABILITY', { required: true }));
             if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) {
                 console.log('No significant data available. Exiting...');
                 return true;
@@ -25018,7 +25019,14 @@ function run() {
             });
             const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded);
             if (shouldFailWorkflow) {
-                core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
+                if (isValidatingStability) {
+                    core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements during the stability checks.
+                    Please rerun the workflow again.
+                    `);
+                }
+                else {
+                    core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
+                }
             }
             return true;
         }

.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts

@@ -19,6 +19,7 @@ async function run() {
         const regressionOutput: CompareResult = JSON.parse(fs.readFileSync('.reassure/output.json', 'utf8'));
         const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', {required: true}));
         const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', {required: true}));
+        const isValidatingStability = Boolean(core.getInput('IS_VALIDATING_STABILITY', {required: true}));
 
         if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) {
             console.log('No significant data available. Exiting...');
@@ -77,7 +78,15 @@ async function run() {
 
         const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded);
         if (shouldFailWorkflow) {
-            core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
+            if (isValidatingStability) {
+                core.setFailed(
+                    `🔴 Duration deviation exceeded the allowed ranges in one or more measurements during the stability checks.
+                    Please rerun the workflow again.
+                    `,
+                );
+            } else {
+                core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
+            }
         }
 
         return true;

.github/workflows/reassurePerfTests.yml

@@ -25,18 +25,26 @@ jobs:
           node-version-file: '.nvmrc'
 
       - name: Install dependencies
-        run: npm install
+        run: npm ci
 
-      - name: Run Reassure baseline tests
-        run: npx reassure --baseline --verbose
+      - name: Reassure stability check
+        run: npx reassure check-stability --verbose
+
+      - name: Validate Reassure stability results
+        id: validateReassureStabilityResults
+        uses: ./.github/actions/javascript/validateReassureOutput
+        with:
+          ALLOWED_DURATION_DEVIATION: 10
+          ALLOWED_RELATIVE_DURATION_DEVIATION: 20
+          IS_VALIDATING_STABILITY: true
 
       - name: Checkout PR head SHA
         run: |
           git fetch origin ${{ github.event.pull_request.head.sha }} --no-tags --depth=1
           git switch --force --detach ${{ github.event.pull_request.head.sha }}
           
       - name: Reinstall dependencies
-        run: npm install --force
+        run: npm ci
 
       - name: Run Reassure delta tests
         run: npx reassure --branch --verbose
@@ -49,9 +57,10 @@ jobs:
           if-no-files-found: ignore
           include-hidden-files: true
 
-      - name: Validate output.json
-        id: validateReassureOutput
+      - name: Validate Reassure results
+        id: validateReassureResults
         uses: ./.github/actions/javascript/validateReassureOutput
         with:
           ALLOWED_DURATION_DEVIATION: 10
           ALLOWED_RELATIVE_DURATION_DEVIATION: 20
+          IS_VALIDATING_STABILITY: false