improve: sharper Verrou dd_line attribution via source filtering and function scoping

Three changes to reduce spurious dd_line reports pointing at do-loop
boundaries, #:endfor lines, or inlined-function call sites:

1. _CONTROL_FLOW_RE + _is_arithmetic_loc: post-filter dd_line results
   by reading the actual source line; skip pure control-flow delimiters
   (end do, end if, #:endfor, $:END_GPU_PARALLEL_LOOP, etc.) that share
   DWARF info with the preceding arithmetic due to template expansion or
   inlining.

2. restrict_syms in _write_dd_run_sh / _run_dd_line: when dd_sym has
   already identified responsible functions, dd_line_run.sh embeds
   --source-function=<sym> flags so Verrou only perturbs FP ops inside
   those functions, narrowing the bisection space.

3. FFLAGS=-fno-inline in fp-stability.yml: prevents gfortran from
   inlining small helpers into their callers, so DWARF correctly
   attributes arithmetic to the callee source line rather than the
   caller loop header.

Author: sbryngelson

.github/workflows/fp-stability.yml

@@ -101,6 +101,12 @@ jobs:
         run: ~/.local/verrou/bin/valgrind --version
 
       - name: Build MFC (debug, serial)
+        # FFLAGS=-fno-inline prevents gfortran from inlining small functions into
+        # their callers.  Without it, DWARF debug info attributes inlined ops to
+        # the caller's line (often a do-loop header), making Verrou dd_line point
+        # to loop boundaries instead of the actual arithmetic.
+        env:
+          FFLAGS: "-fno-inline"
         run: ./mfc.sh build -t pre_process simulation --no-mpi --debug -j"$(nproc)"
 
       - name: Run FP Stability Suite

toolchain/mfc/fp_stability.py

@@ -79,6 +79,55 @@
 # Matches the first "at" frame in a Valgrind stack trace: "(file.fpp:LINE)".
 _VGFRAME_RE = re.compile(r"\(([^):]+\.(?:fpp|f90|F90|c|cpp))\s*:(\d+)\)")
 
+# Lines that are clearly control-flow delimiters rather than arithmetic.
+# dd_line sometimes reports these when the responsible arithmetic is on the
+# preceding line but shares DWARF debug info with the delimiter (e.g. loop
+# boundaries in #:for-expanded code, or inlined functions at call sites).
+_CONTROL_FLOW_RE = re.compile(
+    r"^\s*("
+    r"end\s+(do|if|select|where|forall|subroutine|function|module|program|block)\b"
+    r"|do\s+\w+\s*=\s*[\w,\s]+"  # naked do-loop header (no arithmetic)
+    r"|\$:END_GPU\w+"  # fypp GPU macro closers
+    r"|#:end\w*"  # fypp directive closers (#:endfor, #:enddef, etc.)
+    r"|\s*!\s*$"  # comment-only lines
+    r"|\s*$"  # blank lines
+    r")",
+    re.IGNORECASE,
+)
+
+
+def _read_source_line(fname: str, lineno: int) -> str:
+    """Return the raw source line at lineno (1-based), or '' if unavailable."""
+    if os.path.isabs(fname) and os.path.isfile(fname):
+        candidates = [fname]
+    else:
+        candidates = glob.glob(os.path.join(MFC_ROOT_DIR, "src", "**", os.path.basename(fname)), recursive=True)
+    if not candidates:
+        return ""
+    try:
+        with open(candidates[0]) as fh:
+            lines = fh.readlines()
+        return lines[lineno - 1] if 0 < lineno <= len(lines) else ""
+    except OSError:
+        return ""
+
+
+def _is_arithmetic_loc(fname: str, start: int, end: int) -> bool:
+    """Return True if any line in [start, end] contains non-trivial arithmetic.
+
+    Filters out loop delimiters and fypp directive lines that dd_line sometimes
+    reports when the responsible arithmetic shares DWARF info with its enclosing
+    control-flow boundary (inlining, #:for template expansion, etc.).
+    Returns True (keep) when uncertain so we never silently drop real hotspots.
+    """
+    for lineno in range(start, end + 1):
+        line = _read_source_line(fname, lineno)
+        if not line:
+            return True  # can't read — keep to be safe
+        if not _CONTROL_FLOW_RE.match(line):
+            return True
+    return False
+
 
 def _get_source_context(fname: str, lineno: int, context: int = 2) -> str:
     """Return a annotated source snippet around lineno, or '' if file not found.
@@ -649,7 +698,7 @@ def _run_vprec_sweep(case: dict, verrou_bin: str, sim_bin: str, work_dir: str, r
     return results
 
 
-def _write_dd_run_sh(path: str, verrou_bin: str, sim_bin: str, ic_dir: str):
+def _write_dd_run_sh(path: str, verrou_bin: str, sim_bin: str, ic_dir: str, restrict_syms: list = None):
     """Generate dd_run.sh for verrou_dd_sym / verrou_dd_line.
 
     verrou_dd_* calls: dd_run.sh RUNDIR and injects function/line exclusion via
@@ -659,7 +708,15 @@ def _write_dd_run_sh(path: str, verrou_bin: str, sim_bin: str, ic_dir: str):
     environment — we honour that so the reference is a stable nearest-rounding baseline
     to compare against.  CLI --rounding-mode would override the env var and break the
     reference, so we pass the mode via ${VERROU_ROUNDING_MODE:-float} instead.
+
+    restrict_syms: when provided (a list of function names from dd_sym), the script
+    adds --source-function flags so Verrou only perturbs FP ops inside those functions.
+    This narrows dd_line's search and avoids spurious template-expansion attributions.
     """
+    sym_flags = ""
+    if restrict_syms:
+        sym_flags = " ".join(f"--source-function={s!r}" for s in restrict_syms)
+
     content = textwrap.dedent(f"""\
         #!/usr/bin/env bash
         # Generated by mfc.sh fp-stability — do not edit by hand.
@@ -684,7 +741,7 @@ def _write_dd_run_sh(path: str, verrou_bin: str, sim_bin: str, ic_dir: str):
         ROUND="${{VERROU_ROUNDING_MODE:-float}}"
 
         cd "$TMPDIR_RUN"
-        "$VERROU_BIN" --tool=verrou --error-limit=no --rounding-mode="$ROUND" "$SIM_BIN"
+        "$VERROU_BIN" --tool=verrou --error-limit=no --rounding-mode="$ROUND" {sym_flags} "$SIM_BIN"
         rc=$?
 
         [ -d "$TMPDIR_RUN/D" ] && cp -a "$TMPDIR_RUN/D/." "$RUNDIR/"
@@ -741,10 +798,17 @@ def _dd_env(verrou_bin: str) -> dict:
 
 
 def _parse_rddmin_locs(summary_path: str) -> list:
-    """Extract [(rel_path, start_line, end_line)] from a dd_line rddmin_summary."""
+    """Extract [(rel_path, start_line, end_line)] from a dd_line rddmin_summary.
+
+    Filters out locations whose source lines are pure control-flow delimiters
+    (loop boundaries, fypp directive closers, blank/comment lines).  These can
+    appear when the responsible arithmetic shares DWARF debug info with an
+    enclosing boundary due to inlining or #:for template expansion.
+    """
     if not os.path.isfile(summary_path):
         return []
     locs = []
+    skipped = []
     with open(summary_path) as fh:
         for line in fh:
             m = _LOC_RE.search(line)
@@ -759,7 +823,14 @@ def _parse_rddmin_locs(summary_path: str) -> list:
                     rel = path
             except ValueError:
                 rel = path
-            locs.append((rel.replace("\\", "/"), start, end))
+            rel = rel.replace("\\", "/")
+            if _is_arithmetic_loc(path, start, end):
+                locs.append((rel, start, end))
+            else:
+                skipped.append((rel, start, end))
+    for rel, start, end in skipped:
+        loc = f"{rel}:{start}" if start == end else f"{rel}:{start}-{end}"
+        cons.print(f"  [dim]dd_line: skipped control-flow boundary {loc}[/dim]")
     return locs
 
 
@@ -821,24 +892,37 @@ def _run_dd_sym(case: dict, verrou_bin: str, sim_bin: str, work_dir: str, log_di
     return _parse_rddmin_syms(os.path.join(dd_dir, "dd.sym", "rddmin_summary"))
 
 
-def _run_dd_line(case: dict, verrou_bin: str, sim_bin: str, work_dir: str, log_dir: str, threshold: float = None) -> list:
-    """Run verrou_dd_line; return list of (rel_path, start_line, end_line) tuples."""
+def _run_dd_line(
+    case: dict,
+    verrou_bin: str,
+    sim_bin: str,
+    work_dir: str,
+    log_dir: str,
+    threshold: float = None,
+    restrict_syms: list = None,
+) -> list:
+    """Run verrou_dd_line; return list of (rel_path, start_line, end_line) tuples.
+
+    restrict_syms: function names from dd_sym.  When provided, dd_run.sh passes
+    --source-function=<sym> flags to Verrou so perturbation is restricted to those
+    functions.  This narrows the bisection space and avoids spurious attributions
+    to template-shared or inlined code outside the responsible functions.
+    """
     dd_bin = _find_dd_line(verrou_bin)
     if not dd_bin:
         cons.print("  [dim]verrou_dd_line not found; skipping line-level debug[/dim]")
         return []
 
     dd_dir = os.path.join(log_dir, case["name"])
     os.makedirs(dd_dir, exist_ok=True)
-    dd_run_sh = os.path.join(dd_dir, "dd_run.sh")
+    # Always write a fresh dd_run.sh for dd_line so --source-function flags are included.
+    dd_run_sh = os.path.join(dd_dir, "dd_line_run.sh")
     dd_cmp_py = os.path.join(dd_dir, "dd_cmp.py")
     effective_threshold = threshold if threshold is not None else case["threshold"]
-    if not os.path.isfile(dd_run_sh):
-        _write_dd_run_sh(dd_run_sh, verrou_bin, sim_bin, work_dir)
-        _write_dd_cmp_py(dd_cmp_py, case["compare"], effective_threshold)
-    else:
-        # dd_sym already wrote dd_cmp.py with its threshold; rewrite with ours if different
-        _write_dd_cmp_py(dd_cmp_py, case["compare"], effective_threshold)
+    _write_dd_run_sh(dd_run_sh, verrou_bin, sim_bin, work_dir, restrict_syms=restrict_syms)
+    _write_dd_cmp_py(dd_cmp_py, case["compare"], effective_threshold)
+    if restrict_syms:
+        cons.print(f"  [dim]dd_line restricted to {len(restrict_syms)} dd_sym function(s)[/dim]")
     _run_dd_tool(dd_bin, dd_dir, dd_run_sh, dd_cmp_py, _dd_env(verrou_bin), "dd_line.log", "dd.line", "verrou_dd_line")
     return _parse_rddmin_locs(os.path.join(dd_dir, "dd.line", "rddmin_summary"))
 
@@ -954,7 +1038,19 @@ def _run_case(
                 cons.print(f"  [bold yellow]dd_sym error[/bold yellow]: {exc}")
         if dd_threshold > 0 and run_dd_line:
             try:
-                result["dd_line_locs"] = _run_dd_line(case, verrou_bin, sim_bin, work_dir, log_dir, threshold=dd_threshold)
+                # Seed dd_line with dd_sym results so perturbation is restricted
+                # to the responsible functions, avoiding spurious attributions from
+                # template-expansion or inlining outside those functions.
+                restrict = result["dd_sym_syms"] if result["dd_sym_syms"] else None
+                result["dd_line_locs"] = _run_dd_line(
+                    case,
+                    verrou_bin,
+                    sim_bin,
+                    work_dir,
+                    log_dir,
+                    threshold=dd_threshold,
+                    restrict_syms=restrict,
+                )
             except Exception as exc:
                 cons.print(f"  [bold yellow]dd_line error[/bold yellow]: {exc}")