diff --git a/.github/actions/javascript/validateReassureOutput/action.yml b/.github/actions/javascript/validateReassureOutput/action.yml index 524efa41a..45ea6e81a 100644 --- a/.github/actions/javascript/validateReassureOutput/action.yml +++ b/.github/actions/javascript/validateReassureOutput/action.yml @@ -7,6 +7,9 @@ inputs: ALLOWED_RELATIVE_DURATION_DEVIATION: description: Allowable percentage deviation for the mean duration in regression test results. required: true + IS_VALIDATING_STABILITY: + description: Whether the workflow is validating a Reassure stability check or not. + required: true runs: using: 'node20' main: './index.js' diff --git a/.github/actions/javascript/validateReassureOutput/index.js b/.github/actions/javascript/validateReassureOutput/index.js index b261797ea..822335218 100644 --- a/.github/actions/javascript/validateReassureOutput/index.js +++ b/.github/actions/javascript/validateReassureOutput/index.js @@ -24972,6 +24972,7 @@ function run() { const regressionOutput = JSON.parse(fs_1.default.readFileSync('.reassure/output.json', 'utf8')); const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', { required: true })); const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', { required: true })); + const isValidatingStability = Boolean(core.getInput('IS_VALIDATING_STABILITY', { required: true })); if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) { console.log('No significant data available. Exiting...'); return true; @@ -25018,7 +25019,14 @@ function run() { }); const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded); if (shouldFailWorkflow) { - core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`); + if (isValidatingStability) { + core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements during the stability checks. + Please rerun the workflow again. + `); + } + else { + core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`); + } } return true; } diff --git a/.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts b/.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts index 198b30a1d..5c706ac69 100644 --- a/.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts +++ b/.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts @@ -19,6 +19,7 @@ async function run() { const regressionOutput: CompareResult = JSON.parse(fs.readFileSync('.reassure/output.json', 'utf8')); const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', {required: true})); const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', {required: true})); + const isValidatingStability = Boolean(core.getInput('IS_VALIDATING_STABILITY', {required: true})); if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) { console.log('No significant data available. Exiting...'); @@ -77,7 +78,15 @@ async function run() { const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded); if (shouldFailWorkflow) { - core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`); + if (isValidatingStability) { + core.setFailed( + `🔴 Duration deviation exceeded the allowed ranges in one or more measurements during the stability checks. + Please rerun the workflow again. + `, + ); + } else { + core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`); + } } return true; diff --git a/.github/workflows/reassurePerfTests.yml b/.github/workflows/reassurePerfTests.yml index 946803fca..474c38170 100644 --- a/.github/workflows/reassurePerfTests.yml +++ b/.github/workflows/reassurePerfTests.yml @@ -25,10 +25,18 @@ jobs: node-version-file: '.nvmrc' - name: Install dependencies - run: npm install + run: npm ci - - name: Run Reassure baseline tests - run: npx reassure --baseline --verbose + - name: Reassure stability check + run: npx reassure check-stability --verbose + + - name: Validate Reassure stability results + id: validateReassureStabilityResults + uses: ./.github/actions/javascript/validateReassureOutput + with: + ALLOWED_DURATION_DEVIATION: 10 + ALLOWED_RELATIVE_DURATION_DEVIATION: 20 + IS_VALIDATING_STABILITY: true - name: Checkout PR head SHA run: | @@ -36,7 +44,7 @@ jobs: git switch --force --detach ${{ github.event.pull_request.head.sha }} - name: Reinstall dependencies - run: npm install --force + run: npm ci - name: Run Reassure delta tests run: npx reassure --branch --verbose @@ -49,9 +57,10 @@ jobs: if-no-files-found: ignore include-hidden-files: true - - name: Validate output.json - id: validateReassureOutput + - name: Validate Reassure results + id: validateReassureResults uses: ./.github/actions/javascript/validateReassureOutput with: ALLOWED_DURATION_DEVIATION: 10 ALLOWED_RELATIVE_DURATION_DEVIATION: 20 + IS_VALIDATING_STABILITY: false