Bug 2042834 - Mark unexpected process crashes during mochitest as test failures, r=ahal.

Bug 1631828 inadvertently broadened the synthetic test_end emitted on
process crash so that expected="CRASH" was set unconditionally, marking
every crash as expected. Restrict that to the crashAsPass path so any
other crash takes the elif branch (expected="PASS", status=1) and
registers as an unexpected failure.

Differential Revision: https://phabricator.services.mozilla.com/D302810

Author: fqueze

testing/mochitest/runtests.py

@@ -3138,13 +3138,12 @@ def timeoutHandler():
             )
 
             expected = None
-            if crashAsPass or crash_count > 0:
+            if crashAsPass:
                 # self.message_logger.is_test_running indicates we need a test_end message
                 if self.message_logger.is_test_running:
                     # this works for browser-chrome, mochitest-plain has status=0
                     expected = "CRASH"
-                if crashAsPass:
-                    status = 0
+                status = 0
             elif crash_count or zombieProcesses:
                 if self.message_logger.is_test_running:
                     expected = "PASS"

testing/mochitest/tests/python/test_mochitest_integration.py

@@ -5,6 +5,7 @@
 import os
 from functools import partial
 
+import mozinfo
 import mozunit
 import pytest
 from conftest import setup_args
@@ -214,6 +215,18 @@ def test_output_crash(flavor, runFailures, runtests, test_name):
         if flavor == "browser-chrome":
             results["tbpl_status"] = TBPL_SUCCESS
             results["log_level"] = (INFO, WARNING)
+    elif flavor == "browser-chrome" and not mozinfo.info["debug"]:
+        # A browser-chrome crash isn't matched by mozharness's error regexes
+        # (see the MOZ_CRASHREPORTER_SHUTDOWN note above), so it only registers
+        # as the unexpected CRASH status, which is scored as a warning rather
+        # than a failure.
+        #
+        # This does not apply to debug builds: there the crash also trips the
+        # leak-detection checks (ShutdownLeaks and the DOCSHELL/DOMWINDOW
+        # logging), which emit TEST-UNEXPECTED-FAIL error lines and make the run
+        # a failure, as expected by the default above.
+        results["tbpl_status"] = TBPL_WARNING
+        results["log_level"] = WARNING
 
     status, lines = runtests(
         test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts