i#3538: Fix flakiness in static_maps_mixup_novars (#7959)

The static_maps_mixup_novars is unable to create a fake issue in finding
the DR library bounds on my local machine, and the execution just
completes successfully, not generating the expected error message.

Improves the copy_and_remap logic in static_maps_mixup.c which needs to
break DR's logic for finding library bounds for the purpose of the
static_maps_mixup_novars test, in a way that keeps the
static_maps_mixup_yesvars version still working. We simply add the
writable bit to the first segment mapping to achieve this. With this
there's a hang in debug build.

Adds a guard to get_dynamo_library_bounds to try getting the bounds only
once and during init. This is to prevent a scenario where the 'expected
elf header' ASSERT in memquery_library_bounds fails, which triggers
d_r_internal_error -> report_dynamorio_problem -> privload_print_modules
-> get_dynamorio_dll_start -> get_dynamo_library_bounds again.

Ensures that a failure to find the ELF header is propagated properly in
the release build.

Also adjusts the expected failure message for the debug build, which
actually fails at an ASSERT.

Would be good to observe the test's flakiness over a few days before
removing the "flaky" suffix.

Issue: #3538

Author: abhinav92003

core/unix/memquery.c

@@ -1,5 +1,5 @@
 /* *******************************************************************************
- * Copyright (c) 2010-2025 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2026 Google, Inc.  All rights reserved.
  * Copyright (c) 2011 Massachusetts Institute of Technology  All rights reserved.
  * Copyright (c) 2000-2010 VMware, Inc.  All rights reserved.
  * *******************************************************************************/
@@ -233,6 +233,11 @@ memquery_library_bounds_by_iterator_internal(
                     }
                 } else {
                     ASSERT(false && "expected elf header");
+                    /* Return failure in release build. */
+                    count = 0;
+                    mod_start = NULL;
+                    cur_end = NULL;
+                    break;
                 }
             }
             count++;
@@ -250,7 +255,7 @@ memquery_library_bounds_by_iterator_internal(
      * maps file), but not every library has one.  We have to parse the ELF
      * header to know since we can't assume that a subsequent anonymous
      * region is .bss. */
-    if (image_size != 0 && cur_end - mod_start < image_size) {
+    if (image_size != 0 && cur_end - mod_start < image_size && cur_end != NULL) {
         if (iter.comment[0] != '\0') {
             /* There's something else in the text-data gap: xref i#2641. */
         } else {

core/unix/os.c

@@ -9812,8 +9812,16 @@ extern int dynamorio_so_start, dynamorio_so_end;
 static void
 get_dynamo_library_bounds(void)
 {
+    ASSERT(!dynamo_initialized);
+    static bool attempted_dynamo_library_bounds = false;
     if (dynamorio_library_filepath[0] != '\0') /* Already cached. */
         return;
+    /* Prevent infinite recursion if bounds recovery fails and triggers an assert,
+     * which in turn tries to print modules and query bounds again.
+     */
+    if (attempted_dynamo_library_bounds)
+        return;
+    attempted_dynamo_library_bounds = true;
     /* Note that we're not counting DYNAMORIO_PRELOAD_NAME as a DR area, to match
      * Windows, so we should unload it like we do there. The other reason not to
      * count it is so is_in_dynamo_dll() can be the only exception to the

suite/tests/CMakeLists.txt

@@ -3682,6 +3682,7 @@ if (NOT ANDROID AND NOT APPLE AND NOT RISCV64)
     append_link_flags(api.static_maps_mixup_yesvars
         "-Wl,--defsym,dynamorio_so_start=__executable_start -Wl,--defsym,dynamorio_so_end=end")
 
+    # TODO i#3538: Remove the _FLAKY suffix after ensuring no failures remaining.
     set(api.static_maps_mixup_novars_FLAKY_expectbase
        "static_maps_mixup_novars")
     tobuild_api(api.static_maps_mixup_novars_FLAKY

suite/tests/api/static_maps_mixup.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2019-2026 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -90,7 +90,12 @@ find_exe_bounds(app_pc *base, app_pc *end)
 }
 
 // Confusing the current logic in get_dynamo_library_bounds is as simple as
-// overwriting our first mapping with an anonymously-mapped version.
+// overwriting our first mapping (which contains the ELF header) with an
+// anonymously-mapped version with different protections.
+//
+// By making it anonymous, the segment loses its filename in /proc/self/maps.
+// DR's heuristic will still try to take a valid ELF header under certain
+// conditions; we add the writable bit to prevent that.
 static void
 copy_and_remap(void *base, size_t offs, size_t size)
 {
@@ -100,8 +105,6 @@ copy_and_remap(void *base, size_t offs, size_t size)
     assert(p != MAP_FAILED);
     void *dst = (byte *)base + offs;
     memcpy(p, dst, size);
-    int res = mprotect(p, size, PROT_EXEC | PROT_READ);
-    assert(res == 0);
     void *loc = mremap(p, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, dst);
     assert(loc == dst);
 }

suite/tests/api/static_maps_mixup_novars.templatex

@@ -2,4 +2,8 @@ mix up maps
 remap base=0x[0-9a-f]+, offs=0, sz=[1-9][0-9]*
 pre-DR init
 /* Depending on the environment, sometimes we hit an assert, sometimes a segfault */
-(<Application .*api.static_maps_mixup_novars(_FLAKY)? \([0-9]+\). Failed to find DynamoRIO library bounds.>)?
+#ifdef DEBUG
+.*check failure: .*expected elf header.*
+#else
+.*Failed to find DynamoRIO library bounds.*
+#endif