Merge pull request #133 from devdanzin/oom-dedup-faulthandler-and-family-match

oom_dedup: faulthandler-func SEGV matching + clears-exc family catch-all

Author: devdanzin

fusil/python/oom_dedup.py

@@ -119,7 +119,7 @@ def classify(text):
                 kind="segv", file=None, line=None, func=None, assert_expr=None, fatal_msg=None
             )
         return dict(
-            kind="fatal", file=None, line=None, func=None, assert_expr=None, fatal_msg=msg[:60]
+            kind="fatal", file=None, line=None, func=None, assert_expr=None, fatal_msg=msg
         )
     if SEGV.search(text):
         return dict(kind="segv", file=None, line=None, func=None, assert_expr=None, fatal_msg=None)
@@ -133,7 +133,8 @@ def load_snapshot(lines):
     """Load ``known_sites.tsv`` rows (an iterable of lines) into matcher tables."""
     by_func, by_assert, by_line = {}, {}, {}
     per_file_lines = collections.defaultdict(list)
-    by_msg, kind_of = [], {}
+    by_funcname = {}  # bare func name -> oids (faulthandler-only stacks; see fh_match)
+    by_msg, by_msgfam, kind_of = [], [], {}
     for line in lines:
         line = line.rstrip("\n")
         if not line or line.startswith("#"):
@@ -145,16 +146,22 @@ def load_snapshot(lines):
         kind_of[oid] = kind
         if kt == "func":
             by_func.setdefault(key, set()).add(oid)
+            fn = key.rsplit(":", 1)[-1]  # "file:func" -> "func"
+            if re.fullmatch(r"\w+", fn):  # clean ident only (skip combined "a/b/c(...)" keys)
+                by_funcname.setdefault(fn, set()).add(oid)
         elif kt == "assert":
             by_assert.setdefault(key, set()).add(oid)
         elif kt == "msg":
             by_msg.append((key, oid))
+        elif kt == "msgfam":
+            by_msgfam.append((key, oid))
         elif kt == "line":
             f, ln = key.rsplit(":", 1)
             by_line.setdefault((f, int(ln)), set()).add(oid)
             per_file_lines[f].append((int(ln), oid))
     return dict(
-        func=by_func, assert_=by_assert, line=by_line, fl=per_file_lines, msg=by_msg, kind=kind_of
+        func=by_func, assert_=by_assert, line=by_line, fl=per_file_lines, msg=by_msg,
+        msgfam=by_msgfam, kind=kind_of, funcname=by_funcname,
     )
 
 
@@ -170,19 +177,23 @@ def match(c, snap):
         if hit:
             return hit, "assert"
     if c.get("fatal_msg"):
+        cm = c["fatal_msg"]
         # Match when the cataloged key is a prefix of the crash message (a key may be a short
         # signature, e.g. "_Py_CheckFunctionResult:") OR the (truncation-shortened) crash
-        # message is a prefix of the key. The second clause must use the FULL crash message,
-        # not a fixed [:30] slice -- a short slice stops before the discriminating content
-        # (e.g. "_Py_Dealloc: Deallocator of type '<TYPE>'") and conflates type-specific keys
-        # (OOM-0007 'Context' vs OOM-0023 '_StoreAction'), mislabelling any new type.
-        hit = set(
-            o
-            for k, o in snap["msg"]
-            if c["fatal_msg"].startswith(k) or k.startswith(c["fatal_msg"])
-        )
-        if hit:
-            return hit, "msg"
+        # message is a prefix of the key. Use the FULL crash message, not a fixed [:30] slice --
+        # a short slice stops before the discriminating content (e.g. "_Py_Dealloc: Deallocator
+        # of type '<TYPE>'") and conflates type-specific keys (OOM-0007 'Context' vs OOM-0023
+        # '_StoreAction'). LONGEST match wins so the most specific type key beats a shorter one.
+        exact = [(k, o) for k, o in snap["msg"] if cm.startswith(k) or k.startswith(cm)]
+        if exact:
+            maxlen = max(len(k) for k, _ in exact)
+            return set(o for k, o in exact if len(k) == maxlen), "msg"
+        # Family fallback: a substring identifying a whole bug family (e.g. the generic
+        # subtype_dealloc 'cleared the current exception'), tried ONLY when no type-specific key
+        # matched -> a new/fuzzer type dedups to the family (OOM-0023) instead of oomNEW.
+        fam = set(o for sub, o in snap.get("msgfam", ()) if sub in cm)
+        if fam:
+            return fam, "msgfam"
     if c.get("file") and c.get("func"):
         hit = snap["func"].get("%s:%s" % (c["file"], c["func"]))
         if hit:
@@ -278,6 +289,39 @@ def extract_native_sites(text):
     return out
 
 
+# A faulthandler "Current thread's C stack trace" frame: '... at <func>+0x...'. On a
+# free-threaded debug SEGV this is often the ONLY symbol info (no ASan '#N file.c:line'
+# frames), so extract_native_sites comes back empty.
+_SYM = re.compile(r", at ([A-Za-z_]\w+)\+0x")
+# Funcs to skip when matching such a symbol-only stack (innermost first): the asan/dump/eval/
+# run plumbing + alloc/free + assert detectors + the dealloc dispatch and refcount macros
+# that wrap every dealloc. The first SURVIVING func the catalog keys by name is the site.
+_FH_SKIP = re.compile(
+    r"^(___?interceptor\w*|__sanitizer\w*|__asan\w*|_Py_Dump\w*|faulthandler\w*"
+    r"|_PyEval_EvalFrameDefault|_PyEval_EvalFrame|_PyEval_Vector|PyEval_EvalCode|_PyEval_Frame\w*"
+    r"|Py_RunMain|Py_BytesMain|pymain_\w+|_start|__libc_start\w*|run_mod|run_eval_code_obj"
+    r"|pyrun_\w*|_PyRun_\w*|clear_thread_frame|clear_gen_frame"
+    r"|fatal_error\w*|_Py_FatalError\w*|_PyObject_AssertFailed|_Py_NegativeRefcount"
+    r"|_Py_Dealloc|_Py_MergeZeroLocalRefcount|Py_X?DECREF|Py_X?INCREF|_Py_X?DECREF\w*"
+    r"|_PyMem_Debug\w*|PyMem_\w*Free|PyObject_\w*Free|PyMem_\w*Realloc|PyObject_\w*Realloc"
+    r"|hook_f\w+|tracemalloc_\w+)$"
+)
+
+
+def fh_match(text, snap):
+    """Fallback for a SEGV/generic-fatal whose stdout has a faulthandler C stack (func names)
+    but NO ASan ``#N ... file.c:line`` frames and no gdb resolution. Match the innermost
+    catalog-keyed func BY NAME (e.g. PyList_New -> OOM-0004). Returns (oids, func) or
+    (set(), None)."""
+    for fn in _SYM.findall(text):  # faulthandler prints most-recent-call first
+        if _FH_SKIP.match(fn):
+            continue
+        hit = snap.get("funcname", {}).get(fn)
+        if hit:
+            return set(hit), fn
+    return set(), None
+
+
 def extract_site_from_bt(bt_text):
     """First real CPython frame (back-compat)."""
     sites = extract_sites_from_bt(bt_text)
@@ -439,7 +483,7 @@ def decide(self, stdout_text, source_path=None):
         ]
         if fmsg and not generic_fatal and not fmsg.lower().startswith(("segmentation", "aborted")):
             candidates.append(
-                dict(file=None, line=None, func=None, assert_expr=None, fatal_msg=fmsg[:60])
+                dict(file=None, line=None, func=None, assert_expr=None, fatal_msg=fmsg)
             )
         # Resolve a crash site when the stdout assertion text is unreliable (pure segv /
         # generic-assert fatal) or nothing matched yet. PREFER the native backtrace the
@@ -464,6 +508,13 @@ def decide(self, stdout_text, source_path=None):
         matched = set()
         for c in candidates:
             matched |= match(c, self.snap)[0]
+
+        # Faulthandler-only fallback: a SEGV/generic-fatal with no ASan file:line frames and
+        # no gdb resolution still carries func names in the faulthandler C stack -- match the
+        # innermost catalog-keyed func by name (e.g. PyList_New -> OOM-0004) before giving up.
+        if not matched and not chain and (has_segv or generic_fatal):
+            matched |= fh_match(stdout_text, self.snap)[0]
+
         if matched:
             oid = sorted(matched)[0]
             self.seen[oid] += 1

tests/python/test_oom_dedup.py

@@ -491,5 +491,87 @@ def test_huge_tail_fatal_still_dedupes(self):
         self.assertEqual(label, "OOM-0022")
 
 
+class TestFaulthandlerMatch(unittest.TestCase):
+    """fh_match: SEGVs with only a faulthandler symbol-stack (no ASan file:line frames)
+    resolve by innermost catalog-keyed func name instead of falling to oomSEGV."""
+
+    def setUp(self):
+        fd, self.path = tempfile.mkstemp(suffix=".tsv")
+        os.write(fd, SNAPSHOT.encode())
+        os.close(fd)
+        self.addCleanup(os.remove, self.path)
+        self.snap = oom_dedup.load_snapshot_file(self.path)
+
+    def _segv(self, *funcs):
+        frames = "\n".join(f'  Binary file "python", at {f}+0x10 [0x55]' for f in funcs)
+        return (
+            "Fatal Python error: Segmentation fault\n"
+            "Current thread's C stack trace (most recent call first):\n" + frames + "\n"
+        )
+
+    def test_innermost_keyed_func_wins(self):
+        # detector/eval frames skipped -> code_dealloc (OOM-0003) is the innermost keyed func
+        oids, fn = oom_dedup.fh_match(
+            self._segv("_Py_DumpStack", "_Py_Dealloc", "code_dealloc", "_PyEval_EvalFrameDefault"),
+            self.snap,
+        )
+        self.assertEqual((oids, fn), ({"OOM-0003"}, "code_dealloc"))
+
+    def test_plumbing_and_eval_never_match(self):
+        # _PyEval_EvalFrameDefault IS keyed (OOM-0027) but is generic plumbing -> skipped
+        oids, _ = oom_dedup.fh_match(
+            self._segv("_Py_Dealloc", "Py_DECREF", "_PyEval_EvalFrameDefault", "PyEval_EvalCode"),
+            self.snap,
+        )
+        self.assertEqual(oids, set())
+
+    def test_decide_uses_fh_fallback(self):
+        d = oom_dedup.Deduper(self.path, keep=5)
+        keep, label = d.decide(self._segv("_Py_Dealloc", "dictiter_dealloc"))
+        self.assertEqual(label, "OOM-0006")
+
+    def test_decide_unkeyed_symbol_segv_stays_oomSEGV(self):
+        d = oom_dedup.Deduper(self.path, keep=5)
+        keep, label = d.decide(self._segv("_Py_Dealloc", "some_unkeyed_helper"))
+        self.assertEqual(label, "oomSEGV")
+
+
+class TestMsgFamily(unittest.TestCase):
+    """msgfam catch-all: a new/fuzzer clears-exc type dedups to the family (OOM-0023) while
+    type-specific keys still win (Context->0007, deque->0039), and other invariant variants
+    ('raised'/'overrode') are NOT absorbed."""
+
+    SNAP = "\n".join(
+        [
+            "# oom_id\tkind\tkeytype\tkey",
+            "OOM-0007\tfatal\tmsg\t_Py_Dealloc: Deallocator of type 'Context' cleared the curre",
+            "OOM-0039\tfatal\tmsg\t_Py_Dealloc: Deallocator of type 'collections.deque' cleared",
+            "OOM-0023\tfatal\tmsg\t_Py_Dealloc: Deallocator of type '_StoreAction' cleared the ",
+            "OOM-0023\tfatal\tmsgfam\tcleared the current exception",
+        ]
+    )
+
+    def setUp(self):
+        self.snap = oom_dedup.load_snapshot(self.SNAP.splitlines())
+
+    def _m(self, typ, verb="cleared the current exception"):
+        msg = f"_Py_Dealloc: Deallocator of type '{typ}' {verb}"
+        return oom_dedup.match(dict(fatal_msg=msg), self.snap)[0]
+
+    def test_type_specific_keys_win_over_family(self):
+        self.assertEqual(self._m("Context"), {"OOM-0007"})
+        self.assertEqual(self._m("collections.deque"), {"OOM-0039"})
+        self.assertEqual(self._m("_StoreAction"), {"OOM-0023"})
+
+    def test_new_and_fuzzer_types_fall_back_to_family(self):
+        self.assertEqual(self._m("Evil"), {"OOM-0023"})
+        self.assertEqual(self._m("weird_deque"), {"OOM-0023"})
+        self.assertEqual(self._m("UnknownHandler"), {"OOM-0023"})
+
+    def test_other_invariant_variants_not_absorbed(self):
+        self.assertEqual(self._m("Foo", "raised an exception"), set())
+        self.assertEqual(self._m("Bar", "overrode the current exception"), set())
+
+
 if __name__ == "__main__":
     unittest.main()