Skip to content

Commit c1e04bd

Browse files
EugeniyKiyashkoEugeniyKiyashko
committed
diagnose
1 parent 6cd0e96 commit c1e04bd

5 files changed

Lines changed: 139 additions & 22 deletions

File tree

‎packages/devextreme/docker-ci.sh‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,16 +154,23 @@ function start_runner_watchdog {
154154
local last_suite_time_file="$PWD/testing/LastSuiteTime.txt"
155155
local raw_log_file="$PWD/testing/RawLog.txt"
156156
local last_suite_time=unknown
157+
local stall_count=0
157158

158159
while true; do
159-
sleep 300
160+
sleep 120
160161

161162
if [ ! -f $last_suite_time_file ] || [ $(cat $last_suite_time_file) == $last_suite_time ]; then
162-
echo "Runner stalled"
163-
# tail -n 100 $raw_log_file
164-
# kill -9 $1
163+
stall_count=$((stall_count + 1))
164+
echo "Runner stalled (attempt $stall_count/2)"
165+
166+
if [ $stall_count -ge 2 ]; then
167+
echo "Runner stalled for 10 minutes, killing process..."
168+
tail -n 100 $raw_log_file
169+
kill -9 $1
170+
fi
165171
else
166172
last_suite_time=$(cat $last_suite_time_file)
173+
stall_count=0
167174
fi
168175
done &
169176
}

‎packages/devextreme/testing/runner/Tools/UIModelHelper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ namespace Runner.Tools
1414
public class UIModelHelper
1515
{
1616
// constellation is a set of categories, they are defined in __meta.json files inside category directories
17-
static readonly ICollection<string> KnownConstellations = new HashSet<string> { "export", "misc", "ui", "ui.widgets", "ui.editors", "ui.htmlEditor", "ui.grid", "ui.scheduler", "viz" };
17+
static readonly ICollection<string> KnownConstellations = new HashSet<string> { "export", "misc", "ui", "ui.widgets", "ui.editors", "ui.grid", "ui.scheduler", "viz" };
1818

1919
UrlHelper UrlHelper;
2020
string TestsRootPath;

‎packages/devextreme/testing/runner/Views/Main/RunAll.cshtml‎

Lines changed: 125 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
6262
var TEST_TIMEOUT_SECONDS = 45,
6363
TEST_TIMEOUT = TEST_TIMEOUT_SECONDS * 1000,
64+
WORKER_STUCK_TIMEOUT = 120 * 1000, // 2 minutes without any activity
6465
WORKER_NAME_PREFIX = "workerFrame",
6566
busyCount = 0,
6667
suitesDescription = {
@@ -69,6 +70,7 @@
6970
version: "@(Model.Version)"
7071
},
7172
suitesInProgress = [ ],
73+
workerLastActivity = [ ], // Track last activity time for each worker
7274
urls = @Html.Raw(Json.Serialize(Model.Suites)),
7375
originalUrls = urls.slice(0),
7476
noTryCatch = @Html.Raw(Json.Serialize(Model.NoTryCatch)),
@@ -87,12 +89,8 @@
8789
if(runWorkerInNewWindow)
8890
return 1; // NOTE: more than 1 window cause problems with tests that use focus
8991
90-
var ua = navigator.userAgent;
91-
92-
if(window.ActiveXObject !== undefined)
93-
return 1;
94-
95-
return 2;
92+
// Temporarily reduced to 1 worker to debug stalling issues
93+
return 4; // Was: return 2;
9694
};
9795
9896
WORKER_COUNT = calcWorkerFrameCount();
@@ -145,14 +143,41 @@
145143
var resultSaving = false;
146144
147145
var nextUrl = function(i) {
146+
console.log('nextUrl: worker=' + i + ', urls.length=' + urls.length + ', busyCount=' + busyCount +
147+
', currentSuite=' + (suitesInProgress[i] ? suitesInProgress[i].name : 'null'));
148+
149+
// Safety check: if worker is still busy, do not load next test
150+
if(suitesInProgress[i] !== null && suitesInProgress[i] !== undefined) {
151+
console.error('nextUrl: worker ' + i + ' is still busy with ' + suitesInProgress[i].name + '! Skipping.');
152+
return;
153+
}
154+
148155
if(!urls.length) {
156+
console.log('nextUrl: No more URLs for worker ' + i);
149157
// $.ajax(ROOT_URL + "run/something/FrameHasFinishedRunningASuite.js?frame=" + i);
150158
if(!resultSaving && !busyCount) {
159+
console.log('All tests completed, saving results...');
151160
resultSaving = true;
152161
rootSuite.time = roundTime((new Date() - rootStartTime) / 1000);
153162
rootSuite.pureTime = roundTime(rootSuite.pureTime);
154163
saveResults();
155164
window.onbeforeunload = $.noop;
165+
} else {
166+
console.log('Waiting for tests to complete: resultSaving=' + resultSaving + ', busyCount=' + busyCount);
167+
if(busyCount < 0) {
168+
console.error('ERROR: busyCount is negative! This should not happen.');
169+
}
170+
171+
// Debug: show which workers are still busy
172+
var busyWorkers = [];
173+
for(var w = 0; w < WORKER_COUNT; w++) {
174+
if(suitesInProgress[w]) {
175+
busyWorkers.push('Worker' + w + ':' + suitesInProgress[w].name);
176+
}
177+
}
178+
if(busyWorkers.length > 0) {
179+
console.log('Busy workers: ' + busyWorkers.join(', '));
180+
}
156181
}
157182
return;
158183
}
@@ -191,8 +216,17 @@
191216
192217
startTime: new Date(),
193218
pureTime: 0,
219+
finalized: false, // Track if suite has been finalized
194220
195221
finalize: function(success) {
222+
if(this.finalized) {
223+
console.warn('Suite already finalized: ' + this.name + ', skipping duplicate finalize');
224+
return;
225+
}
226+
227+
var suiteName = this.name; // Save name before cleanup
228+
229+
this.finalized = true;
196230
this.time = roundTime((new Date() - this.startTime) / 1000);
197231
this.pureTime = roundTime(this.pureTime);
198232
delete this.startTime;
@@ -203,14 +237,23 @@
203237
rootSuite.results.push(this);
204238
suitesInProgress[i] = null;
205239
busyCount--;
206-
207-
setTimeout(function() { nextUrl.call(that, _i); }, 0);
240+
241+
console.log('Suite finalized: ' + suiteName + ', worker=' + _i + ', busyCount=' + busyCount +
242+
', scheduling nextUrl in worker ' + _i);
243+
244+
setTimeout(function() {
245+
console.log('Calling nextUrl for worker ' + _i + ' after finalizing ' + suiteName);
246+
nextUrl.call(that, _i);
247+
}, 0);
208248
}
209249
};
210250
211251
worker.name = WORKER_NAME_PREFIX + i;
212-
worker.location = urlInfo.Url + "?" + $.param(additionalParams);
213252
busyCount++;
253+
console.log('Loading test in worker ' + i + ': ' + urlInfo.FullName + ', busyCount=' + busyCount +
254+
', urls remaining=' + urls.length);
255+
worker.location = urlInfo.Url + "?" + $.param(additionalParams);
256+
workerLastActivity[i] = Date.now(); // Mark worker as active
214257
};
215258
216259
var workers = [ ];
@@ -250,6 +293,37 @@
250293
return workers[index];
251294
};
252295
296+
var checkStuckWorkers = function() {
297+
var now = Date.now();
298+
var activeWorkers = [];
299+
300+
for(var i = 0; i < WORKER_COUNT; i++) {
301+
var lastActivity = workerLastActivity[i];
302+
var suite = suitesInProgress[i];
303+
304+
if(suite && !suite.finalized) {
305+
var inactiveSeconds = Math.round((now - lastActivity) / 1000);
306+
activeWorkers.push('Worker' + i + ':' + suite.name + '(' + inactiveSeconds + 's)');
307+
308+
if(lastActivity && (now - lastActivity) > WORKER_STUCK_TIMEOUT) {
309+
console.error('Worker ' + i + ' is STUCK on test: ' + suite.name + ' (no activity for ' +
310+
inactiveSeconds + ' seconds), busyCount=' + busyCount);
311+
console.log('Force finalizing stuck worker ' + i);
312+
313+
// Force finalize the stuck suite (finalize checks for double-finalization)
314+
suite.finalize(false); // Mark as failed
315+
316+
// Reset worker state
317+
workerLastActivity[i] = now;
318+
}
319+
}
320+
}
321+
322+
if(activeWorkers.length > 0 || busyCount > 0) {
323+
console.log('checkStuckWorkers: busyCount=' + busyCount + ', active: [' + activeWorkers.join(', ') + ']');
324+
}
325+
};
326+
253327
var indexFromWorkerName = function(worker) {
254328
return Number(worker.name.substr(WORKER_NAME_PREFIX.length));
255329
};
@@ -276,6 +350,7 @@
276350
var i = indexFromWorkerName(worker),
277351
testSuite = suitesInProgress[i];
278352
353+
workerLastActivity[i] = Date.now(); // Mark worker activity
279354
notifyIsAlive();
280355
281356
var testCase = {
@@ -329,7 +404,10 @@
329404
);
330405
}
331406
332-
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyTestStarted"))), { name: getTestCaseName(testSuite, qunitData) });
407+
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyTestStarted"))), { name: getTestCaseName(testSuite, qunitData) })
408+
.fail(function(jqXHR, textStatus, errorThrown) {
409+
console.warn('NotifyTestStarted failed:', textStatus, errorThrown);
410+
});
333411
};
334412
335413
var indicateTestStatusInTitle = function(failed) {
@@ -384,6 +462,8 @@
384462
testCases,
385463
testCase;
386464
465+
workerLastActivity[i] = Date.now(); // Mark worker activity
466+
387467
// Always notify on test done (removed throttling to prevent stalling)
388468
notifyDeviceTestManager("QUnit.testCaseDone");
389469
notifyIsAlive();
@@ -416,16 +496,37 @@
416496
);
417497
}
418498
419-
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyTestCompleted"))), { name: getTestCaseName(testSuite, qunitData), passed: qunitData.passed === qunitData.total});
499+
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyTestCompleted"))), { name: getTestCaseName(testSuite, qunitData), passed: qunitData.passed === qunitData.total})
500+
.fail(function(jqXHR, textStatus, errorThrown) {
501+
console.warn('NotifyTestCompleted failed:', textStatus, errorThrown);
502+
});
420503
};
421504
422505
window.RUNNER_ON_DONE = function(worker, qunitData) {
423-
var suite = suitesInProgress[indexFromWorkerName(worker)],
506+
var i = indexFromWorkerName(worker),
507+
suite = suitesInProgress[i],
424508
passed = !qunitData.failed;
425509
510+
console.log('RUNNER_ON_DONE: worker=' + i + ', suite=' + (suite ? suite.name : 'null') +
511+
', busyCount=' + busyCount + ', failed=' + qunitData.failed + ', total=' + qunitData.total);
512+
426513
if(suite) {
514+
if(suite.finalized) {
515+
console.error('RUNNER_ON_DONE: suite ' + suite.name + ' is already finalized but still in suitesInProgress[' + i + ']!');
516+
// This should not happen, but clean up just in case
517+
suitesInProgress[i] = null;
518+
return; // Do NOT call nextUrl - finalize() already did
519+
}
520+
521+
var suiteName = suite.name; // Save before finalize clears it
522+
523+
// finalize() handles busyCount-- internally and prevents double-finalization
427524
suite.finalize(passed, qunitData.total);
428-
notifySuiteFinalized(suite.name, passed, qunitData.runtime);
525+
notifySuiteFinalized(suiteName, passed, qunitData.runtime);
526+
} else {
527+
console.warn('RUNNER_ON_DONE: suite is null for worker ' + i + ' - already finalized, skipping nextUrl call');
528+
// Suite is null - it was already finalized and nextUrl was already scheduled
529+
// Do NOT call nextUrl again to avoid loading tests twice in the same worker
429530
}
430531
};
431532
@@ -459,12 +560,21 @@
459560
}
460561
461562
function notifySuiteFinalized(name, passed, runtime) {
462-
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifySuiteFinalized"))), { name: name, passed: passed, runtime: runtime });
563+
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifySuiteFinalized"))), { name: name, passed: passed, runtime: runtime })
564+
.fail(function(jqXHR, textStatus, errorThrown) {
565+
console.warn('NotifySuiteFinalized failed:', textStatus, errorThrown);
566+
});
463567
}
464568
function notifyIsAlive(){
465-
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyIsAlive"))));
569+
$.post(@Html.Raw(Json.Serialize(Url.Action("NotifyIsAlive"))))
570+
.fail(function(jqXHR, textStatus, errorThrown) {
571+
console.warn('NotifyIsAlive failed:', textStatus, errorThrown);
572+
});
466573
}
467574
575+
// Check for stuck workers every 30 seconds
576+
setInterval(checkStuckWorkers, 30000);
577+
468578
function roundTime(time) {
469579
return +(time.toFixed(3));
470580
}

‎packages/devextreme/testing/tests/DevExpress.serverSide.htmlEditor/__meta.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"constellation": "ui.htmlEditor",
2+
"constellation": "ui.editors",
33
"explicit": false,
44
"runOnDevices": false
55
}

‎packages/devextreme/testing/tests/DevExpress.ui.widgets.htmlEditor/__meta.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"constellation": "ui.htmlEditor",
2+
"constellation": "ui.editors",
33
"explicit": false,
44
"runOnDevices": true
55
}

0 commit comments

Comments
 (0)