Skip to content

Commit c2add6c

Browse files
fabioh8010fabioh8010
committed
Improve deviation measurements script
1 parent 58b9c3d commit c2add6c

4 files changed

Lines changed: 56 additions & 37 deletions

File tree

‎.github/actions/javascript/validateReassureOutput/action.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
name: 'Validate Regression Test Output'
22
description: 'Validates the output of regression tests and determines if a test action should fail.'
33
inputs:
4-
DURATION_DEVIATION_PERCENTAGE:
4+
ALLOWED_DURATION_DEVIATION:
5+
description: Allowable deviation in milliseconds for the mean duration in regression test results.
6+
required: true
7+
ALLOWED_RELATIVE_DURATION_DEVIATION:
58
description: Allowable percentage deviation for the mean duration in regression test results.
69
required: true
710
runs:

‎.github/actions/javascript/validateReassureOutput/index.js‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24970,7 +24970,8 @@ function run() {
2497024970
return __awaiter(this, void 0, void 0, function* () {
2497124971
try {
2497224972
const regressionOutput = JSON.parse(fs_1.default.readFileSync('.reassure/output.json', 'utf8'));
24973-
const durationDeviationPercentage = Number(core.getInput('DURATION_DEVIATION_PERCENTAGE', { required: true }));
24973+
const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', { required: true }));
24974+
const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', { required: true }));
2497424975
if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) {
2497524976
console.log('No significant data available. Exiting...');
2497624977
return true;
@@ -24980,40 +24981,44 @@ function run() {
2498024981
for (let i = 0; i < regressionOutput.significant.length; i++) {
2498124982
const index = i + 1;
2498224983
const measurement = regressionOutput.significant[i];
24983-
const baseline = measurement.baseline;
24984-
const current = measurement.current;
24984+
const durationDeviation = measurement.durationDiff;
24985+
const relativeDurationDeviation = measurement.relativeDurationDiff;
24986+
const relativeDurationDeviationPercentage = relativeDurationDeviation * 100;
2498524987
console.log(`Processing measurement ${index}: ${measurement.name}`);
24986-
const isMeasurementRelevant = Math.trunc(current.meanDuration) !== Math.trunc(baseline.meanDuration);
24988+
const isMeasurementRelevant = Math.abs(durationDeviation) > allowedDurationDeviation;
2498724989
if (!isMeasurementRelevant) {
2498824990
console.log(`Skipping measurement ${index} as it's not relevant.`);
2498924991
continue;
2499024992
}
24991-
const increasePercentage = ((current.meanDuration - baseline.meanDuration) / baseline.meanDuration) * 100;
24992-
if (increasePercentage > durationDeviationPercentage) {
24993+
if (relativeDurationDeviationPercentage > durationDeviationPercentage) {
2499324994
outputs.push({
2499424995
name: measurement.name,
24995-
description: `Duration increase percentage exceeded the allowed deviation of ${durationDeviationPercentage}%. Current percentage: ${increasePercentage}%`,
24996-
increasePercentage,
24997-
isGreaterThanDurationDeviationPercentage: true,
24996+
description: `Duration deviation of ${durationDeviation.toFixed(2)} ms (${relativeDurationDeviationPercentage.toFixed(2)}%) exceeded the allowed range of ${allowedDurationDeviation.toFixed(2)} ms (${durationDeviationPercentage.toFixed(2)}%).`,
24997+
relativeDurationDeviationPercentage,
24998+
isDeviationExceeded: true,
2499824999
});
2499925000
}
2500025001
else {
2500125002
outputs.push({
2500225003
name: measurement.name,
25003-
description: `Duration increase percentage ${increasePercentage}% is within the allowed deviation range of ${durationDeviationPercentage}%.`,
25004-
increasePercentage,
25005-
isGreaterThanDurationDeviationPercentage: false,
25004+
description: `Duration deviation of ${durationDeviation.toFixed(2)} ms (${relativeDurationDeviationPercentage.toFixed(2)}%) is within the allowed range of ${allowedDurationDeviation.toFixed(2)} ms (${durationDeviationPercentage.toFixed(2)}%).`,
25005+
relativeDurationDeviationPercentage,
25006+
isDeviationExceeded: false,
2500625007
});
2500725008
}
2500825009
}
25010+
if (outputs.length === 0) {
25011+
console.log('No relevant measurements. Exiting...');
25012+
return true;
25013+
}
2500925014
console.log('\nSummary:');
25010-
outputs.sort((a, b) => b.increasePercentage - a.increasePercentage);
25015+
outputs.sort((a, b) => b.relativeDurationDeviationPercentage - a.relativeDurationDeviationPercentage);
2501125016
outputs.forEach((output) => {
25012-
console.log(`${output.isGreaterThanDurationDeviationPercentage ? '🔴' : '🟢'} ${output.name} > ${output.description}`);
25017+
console.log(`${output.isDeviationExceeded ? '🔴' : '🟢'} ${output.name} > ${output.description}`);
2501325018
});
25014-
const shouldFailWorkflow = outputs.some((output) => output.isGreaterThanDurationDeviationPercentage);
25019+
const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded);
2501525020
if (shouldFailWorkflow) {
25016-
core.setFailed(`🔴 Duration increase percentage exceeded the allowed deviation in one or more measurements.`);
25021+
core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
2501725022
}
2501825023
return true;
2501925024
}

‎.github/actions/javascript/validateReassureOutput/validateReassureOutput.ts‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,21 @@
44
*/
55

66
import * as core from '@actions/core';
7-
import type {CompareResult, MeasureEntry} from '@callstack/reassure-compare';
7+
import type {CompareResult} from '@callstack/reassure-compare';
88
import fs from 'fs';
99

1010
type MeasurementOutput = {
1111
name: string;
1212
description: string;
13-
increasePercentage: number;
14-
isGreaterThanDurationDeviationPercentage: boolean;
13+
relativeDurationDeviationPercentage: number;
14+
isDeviationExceeded: boolean;
1515
};
1616

1717
async function run() {
1818
try {
1919
const regressionOutput: CompareResult = JSON.parse(fs.readFileSync('.reassure/output.json', 'utf8'));
20-
const durationDeviationPercentage = Number(core.getInput('DURATION_DEVIATION_PERCENTAGE', {required: true}));
20+
const allowedDurationDeviation = Number(core.getInput('ALLOWED_DURATION_DEVIATION', {required: true}));
21+
const durationDeviationPercentage = Number(core.getInput('ALLOWED_RELATIVE_DURATION_DEVIATION', {required: true}));
2122

2223
if (regressionOutput.significant === undefined || regressionOutput.significant.length === 0) {
2324
console.log('No significant data available. Exiting...');
@@ -30,44 +31,53 @@ async function run() {
3031
for (let i = 0; i < regressionOutput.significant.length; i++) {
3132
const index = i + 1;
3233
const measurement = regressionOutput.significant[i];
33-
const baseline: MeasureEntry = measurement.baseline;
34-
const current: MeasureEntry = measurement.current;
34+
const durationDeviation = measurement.durationDiff;
35+
const relativeDurationDeviation = measurement.relativeDurationDiff;
36+
const relativeDurationDeviationPercentage = relativeDurationDeviation * 100;
3537

3638
console.log(`Processing measurement ${index}: ${measurement.name}`);
3739

38-
const isMeasurementRelevant = Math.trunc(current.meanDuration) !== Math.trunc(baseline.meanDuration);
40+
const isMeasurementRelevant = Math.abs(durationDeviation) > allowedDurationDeviation;
3941
if (!isMeasurementRelevant) {
4042
console.log(`Skipping measurement ${index} as it's not relevant.`);
4143
continue;
4244
}
4345

44-
const increasePercentage = ((current.meanDuration - baseline.meanDuration) / baseline.meanDuration) * 100;
45-
if (increasePercentage > durationDeviationPercentage) {
46+
if (relativeDurationDeviationPercentage > durationDeviationPercentage) {
4647
outputs.push({
4748
name: measurement.name,
48-
description: `Duration increase percentage exceeded the allowed deviation of ${durationDeviationPercentage}%. Current percentage: ${increasePercentage}%`,
49-
increasePercentage,
50-
isGreaterThanDurationDeviationPercentage: true,
49+
description: `Duration deviation of ${durationDeviation.toFixed(2)} ms (${relativeDurationDeviationPercentage.toFixed(
50+
2,
51+
)}%) exceeded the allowed range of ${allowedDurationDeviation.toFixed(2)} ms (${durationDeviationPercentage.toFixed(2)}%).`,
52+
relativeDurationDeviationPercentage,
53+
isDeviationExceeded: true,
5154
});
5255
} else {
5356
outputs.push({
5457
name: measurement.name,
55-
description: `Duration increase percentage ${increasePercentage}% is within the allowed deviation range of ${durationDeviationPercentage}%.`,
56-
increasePercentage,
57-
isGreaterThanDurationDeviationPercentage: false,
58+
description: `Duration deviation of ${durationDeviation.toFixed(2)} ms (${relativeDurationDeviationPercentage.toFixed(
59+
2,
60+
)}%) is within the allowed range of ${allowedDurationDeviation.toFixed(2)} ms (${durationDeviationPercentage.toFixed(2)}%).`,
61+
relativeDurationDeviationPercentage,
62+
isDeviationExceeded: false,
5863
});
5964
}
6065
}
6166

67+
if (outputs.length === 0) {
68+
console.log('No relevant measurements. Exiting...');
69+
return true;
70+
}
71+
6272
console.log('\nSummary:');
63-
outputs.sort((a, b) => b.increasePercentage - a.increasePercentage);
73+
outputs.sort((a, b) => b.relativeDurationDeviationPercentage - a.relativeDurationDeviationPercentage);
6474
outputs.forEach((output) => {
65-
console.log(`${output.isGreaterThanDurationDeviationPercentage ? '🔴' : '🟢'} ${output.name} > ${output.description}`);
75+
console.log(`${output.isDeviationExceeded ? '🔴' : '🟢'} ${output.name} > ${output.description}`);
6676
});
6777

68-
const shouldFailWorkflow = outputs.some((output) => output.isGreaterThanDurationDeviationPercentage);
78+
const shouldFailWorkflow = outputs.some((output) => output.isDeviationExceeded);
6979
if (shouldFailWorkflow) {
70-
core.setFailed(`🔴 Duration increase percentage exceeded the allowed deviation in one or more measurements.`);
80+
core.setFailed(`🔴 Duration deviation exceeded the allowed ranges in one or more measurements.`);
7181
}
7282

7383
return true;

‎.github/workflows/reassurePerfTests.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,5 @@ jobs:
4545
id: validateReassureOutput
4646
uses: ./.github/actions/javascript/validateReassureOutput
4747
with:
48-
DURATION_DEVIATION_PERCENTAGE: 20
48+
ALLOWED_DURATION_DEVIATION: 10
49+
ALLOWED_RELATIVE_DURATION_DEVIATION: 20

0 commit comments

Comments
 (0)