Skip to content

Commit f7ddbc5

Browse files
EugeniyKiyashkoEugeniyKiyashko
committed
attempt 8
1 parent 2fbfeb5 commit f7ddbc5

13 files changed

Lines changed: 623 additions & 375 deletions

File tree

‎.github/workflows/demos_visual_tests_frameworks.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -388,8 +388,7 @@ jobs:
388388
strategy:
389389
fail-fast: false
390390
matrix:
391-
# CONSTEL: [react(1/4), react(2/4), react(3/4), react(4/4), vue(1/4), vue(2/4), vue(3/4), vue(4/4), angular(1/4), angular(2/4), angular(3/4), angular(4/4)]
392-
CONSTEL: [react(1/6), react(2/6), react(3/6), react(4/6), react(5/6), react(6/6), vue(1/6), vue(2/6), vue(3/6), vue(4/6), vue(5/6), vue(6/6), angular(1/6), angular(2/6), angular(3/6), angular(4/6), angular(5/6), angular(6/6)]
391+
CONSTEL: [react(1/4), react(2/4), react(3/4), react(4/4), vue(1/4), vue(2/4), vue(3/4), vue(4/4), angular(1/4), angular(2/4), angular(3/4), angular(4/4)]
393392
# THEME: ['generic.light', 'material.blue.light', 'fluent.blue.light']
394393
THEME: ['fluent.blue.light']
395394

‎.github/workflows/qunit_tests-renovation.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
6161
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "true"
6262
BUILD_INPROGRESS_RENOVATION: "true"
63-
run: pnpx nx build
63+
run: pnpx nx build:dev
6464

6565
- name: Zip artifacts
6666
working-directory: ./packages/devextreme

‎apps/demos/testing/common.test.js‎

Lines changed: 69 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ import { skippedTests } from './skipped-tests';
2424

2525
import { gitHubIgnored } from '../utils/visual-tests/github-ignored-list';
2626

27+
const testAttempts = {};
28+
2729
const execCode = ClientFunction((code) => {
2830
// eslint-disable-next-line no-eval
2931
const result = eval(code);
@@ -62,10 +64,36 @@ FRAMEWORKS.forEach((approach) => {
6264
if (!shouldRunFramework(approach)) { return; }
6365
fixture(approach)
6466
.beforeEach(async (t) => {
67+
const testKey = `${approach}-${t.testRun.test.name}`;
68+
testAttempts[testKey] = (testAttempts[testKey] || 0) + 1;
69+
70+
console.log(`🔄 Attempt ${testAttempts[testKey]}: ${t.testRun.test.name} (${approach})`);
71+
6572
t.ctx.watchDogHandle = setTimeout(() => { throw new Error('test timeout exceeded'); }, 3 * 60 * 1000);
6673
await t.resizeWindow(1000, 800);
74+
t.ctx.testKey = testKey;
75+
})
76+
.afterEach(async (t) => {
77+
clearTimeout(t.ctx.watchDogHandle);
78+
79+
const testKey = t.ctx.testKey;
80+
const testName = t.testRun.test.name;
81+
const attempts = testAttempts[testKey];
82+
83+
if (t.testRun.errs && t.testRun.errs.length > 0) {
84+
// Test failed
85+
console.log(`❌ Test "${testName}" failed on attempt ${attempts}`);
86+
const errorMsg = t.testRun.errs[0].errMsg || t.testRun.errs[0].message || 'Unknown error';
87+
console.log(` Error: ${errorMsg}`);
88+
} else {
89+
// Test passed
90+
if (attempts > 1) {
91+
console.log(`✅ Test "${testName}" passed on attempt ${attempts}!`);
92+
} else {
93+
console.log(`✅ Test "${testName}" passed on first attempt`);
94+
}
95+
}
6796
})
68-
.afterEach((t) => clearTimeout(t.ctx.watchDogHandle))
6997
.clientScripts([
7098
{ module: 'mockdate' },
7199
// { module: 'axe-core/axe.min.js' },
@@ -124,6 +152,7 @@ FRAMEWORKS.forEach((approach) => {
124152
// pageURL = `http://127.0.0.1:808${getPortByIndex(index)}/Demos/${widgetName}/${demoName}/${approach}/?theme=dx.${theme}`;
125153
// } else {
126154
changeTheme(__dirname, `../${demoPath}/index.html`, process.env.THEME);
155+
console.log(`Running ${testName} ${approach} ${theme} at port ${getPortByIndex(index)}`);
127156
pageURL = `http://127.0.0.1:808${getPortByIndex(index)}/apps/demos/Demos/${widgetName}/${demoName}/${approach}/`;
128157
// }
129158
// // remove when tests enabled not only for datagrid
@@ -141,7 +170,9 @@ FRAMEWORKS.forEach((approach) => {
141170
skipJsErrorsComponents.includes(widgetName),
142171
)
143172
.clientScripts(clientScriptSource)(testName, async (t) => {
144-
console.time('Timer');
173+
const attemptInfo = t.ctx.currentAttempt ? ` (Attempt ${t.ctx.currentAttempt})` : '';
174+
console.log(`[TEST START${attemptInfo}] ${testName} - ${approach} - ${theme}`);
175+
console.time(`Timer-${testName}${attemptInfo}`);
145176

146177
if (visualTestStyles) {
147178
await execCode(visualTestStyles);
@@ -192,7 +223,42 @@ FRAMEWORKS.forEach((approach) => {
192223
await t.expect(comparisonResult).ok('INVALID_SCREENSHOT');
193224
}
194225

195-
console.timeEnd('Timer');
226+
const endAttemptInfo = t.ctx.currentAttempt ? ` (Attempt ${t.ctx.currentAttempt})` : '';
227+
console.log(`[TEST END${endAttemptInfo}] ${testName} - SUCCESS`);
228+
console.timeEnd(`Timer-${testName}${endAttemptInfo}`);
196229
});
197230
});
198231
});
232+
233+
// Test attempts statistics
234+
process.on('exit', () => {
235+
console.log('\n📊 TEST ATTEMPTS STATISTICS:');
236+
237+
const stats = { stable: 0, unstable: 0 };
238+
const unstableTests = [];
239+
240+
Object.entries(testAttempts).forEach(([testKey, attempts]) => {
241+
if (attempts === 1) {
242+
stats.stable++;
243+
} else if (attempts > 1) {
244+
stats.unstable++;
245+
unstableTests.push({ testKey, attempts });
246+
}
247+
});
248+
249+
if (unstableTests.length > 0) {
250+
console.log('\n⚠️ UNSTABLE TESTS:');
251+
unstableTests
252+
.sort((a, b) => b.attempts - a.attempts)
253+
.forEach(({ testKey, attempts }) => {
254+
console.log(` ${testKey}: ${attempts} attempts`);
255+
});
256+
}
257+
258+
console.log(`\n✅ Stable tests: ${stats.stable}`);
259+
console.log(`⚠️ Unstable tests: ${stats.unstable}`);
260+
261+
if (stats.unstable > 0) {
262+
console.log(`\n💡 Tip: Run with TCQUARANTINE=true for automatic retries`);
263+
}
264+
});

‎apps/demos/testing/widgets/common/EditorAppearanceVariants.test.js‎

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -10,39 +10,39 @@ fixture('Common.EditorAppearanceVariants')
1010
});
1111

1212
runManualTest('Common', 'EditorAppearanceVariants', (test) => {
13-
// test('EditorAppearanceVariants', async (t) => {
14-
// const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
15-
16-
// const SELECTBOX_CLASS = 'dx-selectbox';
17-
// const stylingModes = ['outlined', 'filled', 'underlined'];
18-
// const labelModes = ['static', 'floating', 'hidden', 'outside'];
19-
20-
// const clickSaveButton = async () => {
21-
// await t.click($('#validate'));
22-
// };
23-
24-
// const changeLabelMode = async (labelMode) => {
25-
// await t.click($(`.${SELECTBOX_CLASS}`).nth(1));
26-
// await t.click($('.dx-item').withText(labelMode));
27-
// };
28-
29-
// const changeStylingMode = async (stylingMode) => {
30-
// await t.click($(`.${SELECTBOX_CLASS}`).nth(0));
31-
// await t.click($('.dx-item').withText(stylingMode)).wait(500);
32-
// };
33-
34-
// await asyncForEach(stylingModes, async (stylingMode) => {
35-
// await asyncForEach(labelModes, async (labelMode) => {
36-
// await changeStylingMode(stylingMode);
37-
// await changeLabelMode(labelMode);
38-
// await clickSaveButton();
39-
40-
// await testScreenshot(t, takeScreenshot, `common_editor_appearance_variants_${stylingMode}_${labelMode}_desktop.png`);
41-
// });
42-
// });
43-
44-
// await t
45-
// .expect(compareResults.isValid())
46-
// .ok(compareResults.errorMessages());
47-
// });
13+
test('EditorAppearanceVariants', async (t) => {
14+
const { takeScreenshot, compareResults } = createScreenshotsComparer(t);
15+
16+
const SELECTBOX_CLASS = 'dx-selectbox';
17+
const stylingModes = ['outlined', 'filled', 'underlined'];
18+
const labelModes = ['static', 'floating', 'hidden', 'outside'];
19+
20+
const clickSaveButton = async () => {
21+
await t.click($('#validate'));
22+
};
23+
24+
const changeLabelMode = async (labelMode) => {
25+
await t.click($(`.${SELECTBOX_CLASS}`).nth(1));
26+
await t.click($('.dx-item').withText(labelMode));
27+
};
28+
29+
const changeStylingMode = async (stylingMode) => {
30+
await t.click($(`.${SELECTBOX_CLASS}`).nth(0));
31+
await t.click($('.dx-item').withText(stylingMode)).wait(500);
32+
};
33+
34+
await asyncForEach(stylingModes, async (stylingMode) => {
35+
await asyncForEach(labelModes, async (labelMode) => {
36+
await changeStylingMode(stylingMode);
37+
await changeLabelMode(labelMode);
38+
await clickSaveButton();
39+
40+
await testScreenshot(t, takeScreenshot, `common_editor_appearance_variants_${stylingMode}_${labelMode}_desktop.png`);
41+
});
42+
});
43+
44+
await t
45+
.expect(compareResults.isValid())
46+
.ok(compareResults.errorMessages());
47+
});
4848
});

‎apps/demos/utils/visual-tests/attempt-tracker.js‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
class AttemptTracker {
2+
constructor() {
3+
this.attempts = new Map();
4+
this.startTimes = new Map();
5+
}
6+
7+
registerAttempt(testKey, testName, framework) {
8+
const currentAttempt = (this.attempts.get(testKey) || 0) + 1;
9+
this.attempts.set(testKey, currentAttempt);
10+
this.startTimes.set(testKey, Date.now());
11+
12+
const logMessage = `[ATTEMPT ${currentAttempt}] Starting test: ${testName} (${framework})`;
13+
console.log(`\x1b[36m${logMessage}\x1b[0m`);
14+
15+
return currentAttempt;
16+
}
17+
18+
registerSuccess(testKey, testName) {
19+
const attempt = this.attempts.get(testKey) || 1;
20+
const startTime = this.startTimes.get(testKey);
21+
const duration = startTime ? Date.now() - startTime : 0;
22+
23+
const logMessage = `[SUCCESS] Test "${testName}" passed on attempt ${attempt} (${duration}ms)`;
24+
console.log(`\x1b[32m${logMessage}\x1b[0m`);
25+
26+
// if (process.env.TRACK_ATTEMPTS) {
27+
this.writeAttemptStats(testName, attempt, 'SUCCESS', duration);
28+
// }
29+
}
30+
31+
registerFailure(testKey, testName, error) {
32+
const attempt = this.attempts.get(testKey) || 1;
33+
const startTime = this.startTimes.get(testKey);
34+
const duration = startTime ? Date.now() - startTime : 0;
35+
36+
const logMessage = `[FAILURE] Test "${testName}" failed on attempt ${attempt} (${duration}ms): ${error}`;
37+
console.log(`\x1b[31m${logMessage}\x1b[0m`);
38+
39+
// if (process.env.TRACK_ATTEMPTS) {
40+
this.writeAttemptStats(testName, attempt, 'FAILURE', duration, error);
41+
// }
42+
}
43+
44+
getCurrentAttempt(testKey) {
45+
return this.attempts.get(testKey) || 0;
46+
}
47+
48+
writeAttemptStats(testName, attempt, status, duration, error = '') {
49+
const fs = require('fs');
50+
const path = require('path');
51+
52+
const statsFile = path.join(process.cwd(), 'test-attempts.log');
53+
const timestamp = new Date().toISOString();
54+
const logEntry = `${timestamp},${testName},${attempt},${status},${duration}ms,${error}\n`;
55+
56+
fs.appendFileSync(statsFile, logEntry);
57+
}
58+
59+
generateReport() {
60+
const fs = require('fs');
61+
const path = require('path');
62+
63+
const reportFile = path.join(process.cwd(), 'attempts-report.txt');
64+
const report = [];
65+
66+
report.push('=== Test Attempts Report ===\n');
67+
report.push(`Generated at: ${new Date().toISOString()}\n`);
68+
69+
const attemptStats = {};
70+
for (const [testKey, attempts] of this.attempts.entries()) {
71+
if (!attemptStats[attempts]) {
72+
attemptStats[attempts] = 0;
73+
}
74+
attemptStats[attempts]++;
75+
}
76+
77+
report.push('Attempts Distribution:');
78+
for (const [attempts, count] of Object.entries(attemptStats).sort()) {
79+
report.push(` ${attempts} attempt(s): ${count} test(s)`);
80+
}
81+
82+
fs.writeFileSync(reportFile, report.join('\n'));
83+
console.log(`\x1b[33mAttempts report saved to: ${reportFile}\x1b[0m`);
84+
}
85+
}
86+
87+
module.exports = new AttemptTracker();

0 commit comments

Comments
 (0)