i#7157: Inject syscall templates dynamically in scheduler (#7277)

Adds support to the drmemtrace scheduler for injecting system call trace
templates dynamically during scheduling. This obviates the need to
create a separate statically-injected trace with system call trace
templates.

Reuses context switch trace injection code to the extent possible.

Adds a new analyzer flag -sched_syscall_file and new scheduler_options_t
fields to allow specifying the system call trace template file.

Adds a mock system call template file generated using the
burst_syscall_inject test (slightly modified to use sysnums that match
the checked-in test trace).

Adds various unit tests: a new scheduler unit test that verifies dynamic
syscall trace injection and its effect on scheduling, analyzer tests
that use -sched_syscall_file for dynamic injection in core-sharded and
non-core-sharded modes, and an invariant checker test on the added mock
system call template file.

Issue: #7157

Author: abhinav92003

api/docs/release.dox

@@ -141,6 +141,11 @@ Further non-compatibility-affecting changes include:
    instructions in the trace. This works for traces that have embedded instruction
    encodings in them, and also for legacy traces without embedded encodings where the
    encodings are obtained from the application binaries instead.
+ - Added a new drmemtrace analyzer flag -sched_syscall_file to allow specifying the
+   system call trace template file to be used for dynamic injection of system call trace
+   templates. Added similar options for the drmemtrace scheduler: #dynamorio::drmemtrace::
+   scheduler_tmpl_t::scheduler_options_t::kernel_syscall_trace_path, and #dynamorio::
+   drmemtrace::scheduler_tmpl_t::scheduler_options_t::kernel_syscall_reader.
 
 **************************************************
 <hr>

clients/drcachesim/analyzer.cpp

@@ -320,19 +320,22 @@ analyzer_tmpl_t<RecordType, ReaderType>::init_scheduler_common(
             sched_ops.single_lockstep_output = true;
             worker_count_ = 1;
         }
-    } else if (parallel_) {
-        sched_ops = sched_type_t::make_scheduler_parallel_options(verbosity_);
-        sched_ops.replay_as_traced_istream = options.replay_as_traced_istream;
-        sched_ops.read_inputs_in_init = options.read_inputs_in_init;
-        if (worker_count_ <= 0)
-            worker_count_ = std::thread::hardware_concurrency();
-        output_count = worker_count_;
     } else {
-        sched_ops = sched_type_t::make_scheduler_serial_options(verbosity_);
+        if (parallel_) {
+            sched_ops = sched_type_t::make_scheduler_parallel_options(verbosity_);
+            if (worker_count_ <= 0)
+                worker_count_ = std::thread::hardware_concurrency();
+            output_count = worker_count_;
+        } else {
+            sched_ops = sched_type_t::make_scheduler_serial_options(verbosity_);
+            worker_count_ = 1;
+            output_count = 1;
+        }
+        // As noted in the init_scheduler_common() header comment, we preserve only
+        // some select fields.
         sched_ops.replay_as_traced_istream = options.replay_as_traced_istream;
         sched_ops.read_inputs_in_init = options.read_inputs_in_init;
-        worker_count_ = 1;
-        output_count = 1;
+        sched_ops.kernel_syscall_trace_path = options.kernel_syscall_trace_path;
     }
     sched_mapping_ = options.mapping;
     if (scheduler_.init(workloads, output_count, std::move(sched_ops)) !=

clients/drcachesim/analyzer.h

@@ -217,22 +217,25 @@ template <typename RecordType, typename ReaderType> class analyzer_tmpl_t {
         operator=(const analyzer_worker_data_t &) = delete;
     };
 
+    // See comment on init_scheduler_common() for some noteworthy details.
     bool
     init_scheduler(const std::vector<std::string> &trace_paths,
                    // To include all threads/shards, use empty sets.
                    const std::set<memref_tid_t> &only_threads,
                    const std::set<int> &only_shards, int output_limit, int verbosity,
                    typename sched_type_t::scheduler_options_t options);
 
-    // For core-sharded, worker_count_ must be set prior to calling this; for parallel
-    // mode if it is not set it will be set to the underlying core count.
-    // For core-sharded, all of "options" is used; otherwise, only the
-    // read_inputs_in_init field is preserved.
+    // See comment on init_scheduler_common() for some noteworthy details.
     bool
     init_scheduler(std::unique_ptr<ReaderType> reader,
                    std::unique_ptr<ReaderType> reader_end, int verbosity,
                    typename sched_type_t::scheduler_options_t options);
 
+    // For core-sharded, worker_count_ must be set prior to calling this; for parallel
+    // mode if it is not set it will be set to the underlying core count.
+    // For core-sharded, all of "options" is used; otherwise, the
+    // read_inputs_in_init, replay_as_traced_istream, and kernel_syscall_trace_path
+    // fields are preserved.
     bool
     init_scheduler_common(std::vector<typename sched_type_t::input_workload_t> &workloads,
                           typename sched_type_t::scheduler_options_t options);

clients/drcachesim/analyzer_multi.cpp

@@ -571,6 +571,8 @@ analyzer_multi_tmpl_t<RecordType, ReaderType>::analyzer_multi_tmpl_t()
 #endif
     }
 
+    sched_ops.kernel_syscall_trace_path = op_sched_syscall_file.get_value();
+
     if (!indirs.empty()) {
         std::vector<std::string> tracedirs;
         for (const std::string &indir : indirs)

clients/drcachesim/common/memtrace_stream.h

@@ -113,7 +113,7 @@ class memtrace_stream_t {
          */
         SCHED_STAT_HIT_OUTPUT_LIMIT,
         /**
-         * Counts the instances when the kernel context switch sequence was injected.
+         * Counts the instances when the kernel context switch sequences were injected.
          */
         SCHED_STAT_KERNEL_SWITCH_SEQUENCE_INJECTIONS,
         /** Count of statistic types. */

clients/drcachesim/common/options.cpp

@@ -1039,12 +1039,20 @@ droption_t<std::string>
 #endif
 droption_t<std::string> op_sched_switch_file(
     DROPTION_SCOPE_FRONTEND, "sched_switch_file", "",
-    "Path to file holding context switch sequences",
+    "Path to file holding kernel context switch sequences",
     "Applies to -core_sharded and -core_serial.  Path to file holding context switch "
     "sequences.  The file can contain multiple sequences each with regular trace headers "
     "and the sequence proper bracketed by TRACE_MARKER_TYPE_CONTEXT_SWITCH_START and "
     "TRACE_MARKER_TYPE_CONTEXT_SWITCH_END markers.");
 
+droption_t<std::string> op_sched_syscall_file(
+    DROPTION_SCOPE_FRONTEND, "sched_syscall_file", "",
+    "Path to file holding kernel system call sequences",
+    "Path to file holding system call sequences.  The file can contain multiple "
+    "sequences each with regular trace headers and the sequence proper bracketed by "
+    "TRACE_MARKER_TYPE_SYSCALL_TRACE_START and TRACE_MARKER_TYPE_SYSCALL_TRACE_END "
+    "markers.");
+
 droption_t<bool> op_sched_randomize(
     DROPTION_SCOPE_FRONTEND, "sched_randomize", false,
     "Pick next inputs randomly on context switches",

clients/drcachesim/common/options.h

@@ -216,6 +216,7 @@ extern dynamorio::droption::droption_t<std::string> op_replay_file;
 extern dynamorio::droption::droption_t<std::string> op_cpu_schedule_file;
 #endif
 extern dynamorio::droption::droption_t<std::string> op_sched_switch_file;
+extern dynamorio::droption::droption_t<std::string> op_sched_syscall_file;
 extern dynamorio::droption::droption_t<bool> op_sched_randomize;
 extern dynamorio::droption::droption_t<bool> op_sched_disable_direct_switches;
 extern dynamorio::droption::droption_t<bool> op_sched_infinite_timeouts;

clients/drcachesim/scheduler/scheduler.h

@@ -829,6 +829,31 @@ template <typename RecordType, typename ReaderType> class scheduler_tmpl_t {
          * when raising this value on uneven inputs.
          */
         double exit_if_fraction_inputs_left = 0.1;
+        /**
+         * Input file containing template sequences of kernel system call code.
+         * Each sequence must start with a #TRACE_MARKER_TYPE_SYSCALL_TRACE_START
+         * marker and end with #TRACE_MARKER_TYPE_SYSCALL_TRACE_END.
+         * The value of each marker must hold the system call number for the system call
+         * it corresponds to. Sequences for multiple system calls are concatenated into a
+         * single file. Each sequence should be in the regular offline drmemtrace format.
+         * Whenever a #TRACE_MARKER_TYPE_SYSCALL marker is encountered in a trace, if a
+         * corresponding sequence with the same marker value exists it is inserted into
+         * the output stream after the #TRACE_MARKER_TYPE_SYSCALL marker.
+         * The same file (or reader) must be passed when replaying as this kernel
+         * code is not stored when recording.
+         * An alternative to passing the file path is to pass #kernel_syscall_reader
+         * and #kernel_syscall_reader_end.
+         */
+        std::string kernel_syscall_trace_path;
+        /**
+         * An alternative to #kernel_syscall_trace_path is to pass a reader and
+         * #kernel_syscall_reader_end.  See the description of #kernel_syscall_trace_path.
+         * This field is only examined if #kernel_syscall_trace_path is empty.
+         * The scheduler will call the init() function for the reader.
+         */
+        std::unique_ptr<ReaderType> kernel_syscall_reader;
+        /** The end reader for #kernel_syscall_reader. */
+        std::unique_ptr<ReaderType> kernel_syscall_reader_end;
         // When adding new options, also add to print_configuration().
     };
 

clients/drcachesim/scheduler/scheduler_dynamic.cpp

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2023-2024 Google, Inc.  All rights reserved.
+ * Copyright (c) 2023-2025 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -596,16 +596,10 @@ scheduler_dynamic_tmpl_t<RecordType, ReaderType>::process_marker(
     case TRACE_MARKER_TYPE_CONTEXT_SWITCH_START:
         outputs_[output].in_context_switch_code = true;
         break;
-    case TRACE_MARKER_TYPE_SYSCALL_TRACE_START:
-        outputs_[output].in_syscall_code = true;
-        break;
     case TRACE_MARKER_TYPE_CONTEXT_SWITCH_END:
         // We have to delay until the next record.
         outputs_[output].hit_switch_code_end = true;
         break;
-    case TRACE_MARKER_TYPE_SYSCALL_TRACE_END:
-        outputs_[output].in_syscall_code = false;
-        break;
     case TRACE_MARKER_TYPE_TIMESTAMP:
         // Syscall sequences are not expected to have a timestamp.
         assert(!outputs_[output].in_syscall_code);