kernel: force reprogram if kernel device has device global with init_mode reprogram and extend unit test

Author: sophimao

src/acl_kernel.cpp

@@ -3041,23 +3041,33 @@ int acl_submit_kernel_device_op(cl_event event) {
   // to free up old operation slots.
   acl_forget_proposed_device_ops(doq);
 
+  // Force reprogram if there is device global with init_mode:reprogram
+  // or if this is the split kernel mode (split kernel workaround)
   bool need_reprogram = true;
-  if (device->last_bin) {
-    // compare hash of last program that went through device op queue and the
-    // program required by kernel
-    need_reprogram =
-        device->last_bin->get_devdef().autodiscovery_def.binary_rand_hash !=
-        dev_bin->get_devdef().autodiscovery_def.binary_rand_hash;
-  } else {
-    // compare hash of program that is on the device and the program required by
-    // kernel
-    need_reprogram = device->def.autodiscovery_def.binary_rand_hash !=
-                     dev_bin->get_devdef().autodiscovery_def.binary_rand_hash;
-  }
-
-  if (event->context->split_kernel) {
-    // Always reprogram in split kernel mode. This is a temporary workaround.
-    need_reprogram = true;
+  // Else check if reprogram is needed based on hash of loaded binary
+  // First try to find if there are any reprogram device globals
+  std::unordered_map<std::string, acl_device_global_mem_def_t>
+      device_global_mem_defs =
+          device->def.autodiscovery_def.device_global_mem_defs;
+  const auto reprogram_it = std::find_if(
+      device_global_mem_defs.begin(), device_global_mem_defs.end(),
+      [](const auto &it) {
+        return it.second.init_mode == ACL_DEVICE_GLOBAL_INIT_MODE_REPROGRAM;
+      });
+  if (!event->context->split_kernel &&
+      (reprogram_it == device_global_mem_defs.end())) {
+    if (device->last_bin) {
+      // compare hash of last program that went through device op queue and the
+      // program required by kernel
+      need_reprogram =
+          device->last_bin->get_devdef().autodiscovery_def.binary_rand_hash !=
+          dev_bin->get_devdef().autodiscovery_def.binary_rand_hash;
+    } else {
+      // compare hash of program that is on the device and the program required
+      // by kernel
+      need_reprogram = device->def.autodiscovery_def.binary_rand_hash !=
+                       dev_bin->get_devdef().autodiscovery_def.binary_rand_hash;
+    }
   }
 
   if (need_reprogram) {

test/acl_kernel_test.cpp

@@ -3968,7 +3968,7 @@ TEST(acl_kernel_reprogram_scheduler, switch_prog) {
   // set MEM_MIGRATE2.1 to COMPLETE +
   // set MEM_MIGRATE2.2 to RUNNING +
   // set MEM_MIGRATE2.2 to COMPLETE +
-  // submit KERNEL2 to device = 5
+  // submit KERNEL2 to device = 10
   CHECK_EQUAL(offset + 15, m_devlog.num_ops);
 
   // Should have copied the memory over.
@@ -4332,6 +4332,180 @@ TEST(acl_kernel_reprogram_scheduler, switch_prog) {
   CHECK_EQUAL(CL_SUCCESS, clReleaseCommandQueue(cq2));
 }
 
+TEST(acl_kernel_reprogram_scheduler, device_global_reprogram) {
+  // In this test, we will force the device to contain reprogram
+  // device global. The device will be first reprogrammed eagerly
+  // due to the clCreateProgramWithBinary call, then when the
+  // kernel is enqueued, another reprogram should be scheduled
+  // even though the device is already programmed with the right
+  // binary, due to the presence of the device global.
+
+  // Force device to contain device global
+  m_device->def.autodiscovery_def.device_global_mem_defs.insert(
+      {"dev_glob1",
+       {/* address */ 1024,
+        /* size */ 1024,
+        /* host_access */ ACL_DEVICE_GLOBAL_HOST_ACCESS_READ_WRITE,
+        /* init_mode */ ACL_DEVICE_GLOBAL_INIT_MODE_REPROGRAM,
+        /* implement_in_csr */ false}});
+
+  // Initial eager reprogram
+  int offset = m_devlog.num_ops;
+  CHECK_EQUAL(3, offset);
+
+  acl_device_program_info_t *dp0 = check_dev_prog(m_program0);
+
+  m_context->reprogram_buf_read_callback = read_mem_callback;
+  m_context->reprogram_buf_write_callback = write_mem_callback;
+
+  // A device side buffer
+  cl_int status = CL_INVALID_VALUE;
+  cl_mem mem = clCreateBuffer(m_context, CL_MEM_READ_WRITE, 2048, 0, &status);
+  CHECK_EQUAL(CL_SUCCESS, status);
+  CHECK(mem);
+  memset(mem->host_mem.aligned_ptr, 'X', mem->size);
+  memset(mem->block_allocation->range.begin, 'x', mem->size);
+
+  CHECK_EQUAL(1, m_context->device_buffers_have_backing_store);
+  CHECK_EQUAL(0, mem->block_allocation->region->is_host_accessible);
+  CHECK_EQUAL(0, mem->writable_copy_on_host);
+
+  cl_kernel k = get_kernel(m_program0);
+
+  // Just the initial program load.
+  CHECK_EQUAL(m_first_dev_bin, m_device->last_bin);
+  CHECK_EQUAL(m_first_dev_bin, m_device->loaded_bin);
+
+  cl_event ue = get_user_event();
+  cl_event k_e = 0;
+
+  CHECK_EQUAL(CL_SUCCESS, clSetKernelArg(k, 0, sizeof(cl_mem), &mem));
+  CHECK_EQUAL(CL_SUCCESS, clSetKernelArg(k, 1, sizeof(cl_mem), &mem));
+  CHECK_EQUAL(CL_SUCCESS, clEnqueueTask(m_cq, k, 1, &ue, &k_e));
+  CHECK_EQUAL(CL_COMMAND_TASK, k_e->cmd.type);
+
+  // Only initial programming has occurred.
+  // Has 3 transitions logged: SUBMITTED, RUNNING, COMPLETE
+  CHECK_EQUAL(m_first_dev_bin, m_device->last_bin);
+  CHECK_EQUAL(m_first_dev_bin, m_device->loaded_bin);
+
+  acl_print_debug_msg("Forcing user event completion\n");
+  CHECK_EQUAL(CL_SUCCESS, clSetUserEventStatus(ue, CL_COMPLETE));
+  CHECK_EQUAL(CL_SUCCESS, clReleaseEvent(ue));
+
+  // Should have recorded that we loaded the program.
+  CHECK_EQUAL(&(dp0->device_binary), m_device->last_bin);
+  CHECK_EQUAL(&(dp0->device_binary), m_device->loaded_bin);
+
+  // submit device global forced REPROGRAM +
+  // set REPROGRAM to RUNNING +
+  // set REPROGRAM to COMPLETE +
+  // set MEM_MIGRATE 1 to RUNNING +
+  // set MEM_MIGRATE 1 to COMPLETE +
+  // set MEM_MIGRATE 2 to RUNNING +
+  // set MEM_MIGRATE 2 to COMPLETE +
+  // submit KERNEL = 8
+  CHECK_EQUAL(offset + 8, m_devlog.num_ops);
+  const acl_device_op_t *op0submit = &(m_devlog.before[3]);
+  const acl_device_op_t *op0running = &(m_devlog.before[4]);
+  const acl_device_op_t *op0complete = &(m_devlog.before[5]);
+
+  // Device global forced reprogram
+  CHECK_EQUAL(ACL_DEVICE_OP_REPROGRAM, op0submit->info.type);
+  CHECK_EQUAL(0, op0submit->id);
+  CHECK(op0submit->info.event);
+  CHECK_EQUAL(CL_SUBMITTED, op0submit->status);
+  CHECK_EQUAL(0, op0submit->info.num_printf_bytes_pending);
+  CHECK_EQUAL(1, op0submit->first_in_group);
+  CHECK_EQUAL(0, op0submit->last_in_group);
+
+  CHECK_EQUAL(ACL_DEVICE_OP_REPROGRAM, op0running->info.type);
+  CHECK_EQUAL(0, op0running->id);
+  CHECK(op0running->info.event);
+  CHECK_EQUAL(CL_RUNNING, op0running->status);
+  CHECK_EQUAL(0, op0running->info.num_printf_bytes_pending);
+  CHECK_EQUAL(1, op0running->first_in_group);
+  CHECK_EQUAL(0, op0running->last_in_group);
+
+  CHECK_EQUAL(ACL_DEVICE_OP_REPROGRAM, op0complete->info.type);
+  CHECK_EQUAL(0, op0complete->id);
+  CHECK(op0complete->info.event);
+  CHECK_EQUAL(CL_COMPLETE, op0complete->status);
+  CHECK_EQUAL(0, op0complete->info.num_printf_bytes_pending);
+  CHECK_EQUAL(1, op0complete->first_in_group);
+  CHECK_EQUAL(0, op0complete->last_in_group);
+
+  // The device is still programmed with the same program.
+  CHECK_EQUAL(&(dp0->device_binary), m_device->last_bin);
+  CHECK_EQUAL(&(dp0->device_binary), m_device->loaded_bin);
+
+  const acl_device_op_t *op1submit = &(m_devlog.before[10]);
+  CHECK_EQUAL(ACL_DEVICE_OP_KERNEL, op1submit->info.type);
+  CHECK_EQUAL(k_e, op1submit->info.event);
+  CHECK_EQUAL(CL_SUBMITTED, op1submit->status);
+  CHECK_EQUAL(0, op1submit->info.num_printf_bytes_pending);
+  CHECK_EQUAL(0, op1submit->first_in_group); // reprogram is first
+  CHECK_EQUAL(1, op1submit->last_in_group);
+
+  // The user-level event is linked to the kernel device op now.
+  CHECK_EQUAL(op1submit->id, k_e->current_device_op->id);
+
+  // Pretend to start the kernel
+  acl_print_debug_msg("Say kernel is running\n");
+  ACL_LOCKED(acl_receive_kernel_update(k_e->current_device_op->id, CL_RUNNING));
+  CHECK_EQUAL(CL_RUNNING, k_e->current_device_op->execution_status);
+
+  ACL_LOCKED(acl_idle_update(m_context));
+
+  // Now we have a "running" transition
+  CHECK_EQUAL(offset + 9, m_devlog.num_ops);
+  const acl_device_op_t *op2a = &(m_devlog.after[11]);
+  CHECK_EQUAL(ACL_DEVICE_OP_KERNEL, op2a->info.type);
+  CHECK_EQUAL(k_e, op2a->info.event);
+  CHECK_EQUAL(CL_RUNNING, op2a->status);
+  CHECK_EQUAL(0, op2a->info.num_printf_bytes_pending);
+  CHECK_EQUAL(0, op2a->first_in_group);
+  CHECK_EQUAL(1, op2a->last_in_group);
+
+  // The running status was propagated up to the user-level event.
+  CHECK_EQUAL(CL_RUNNING, k_e->execution_status);
+
+  acl_print_debug_msg("Say kernel is complete\n");
+  ACL_LOCKED(
+      acl_receive_kernel_update(k_e->current_device_op->id, CL_COMPLETE));
+  CHECK_EQUAL(CL_COMPLETE, k_e->current_device_op->execution_status);
+
+  ACL_LOCKED(acl_idle_update(m_context));
+  // Now we have a "complete" transition
+  CHECK_EQUAL(offset + 10, m_devlog.num_ops);
+  const acl_device_op_t *op3a = &(m_devlog.after[12]);
+  CHECK_EQUAL(ACL_DEVICE_OP_KERNEL, op3a->info.type);
+  CHECK_EQUAL(k_e, op3a->info.event);
+  CHECK_EQUAL(CL_COMPLETE, op3a->status);
+  CHECK_EQUAL(0, op3a->info.num_printf_bytes_pending);
+  CHECK_EQUAL(0, op3a->first_in_group);
+  CHECK_EQUAL(1, op3a->last_in_group);
+
+  // Completion timestamp has propagated up to the user level event.
+  CHECK_EQUAL(acl_platform.device_op_queue.op[op3a->id].timestamp[CL_COMPLETE],
+              k_e->timestamp[CL_COMPLETE]);
+
+  // Completion wipes out the downlink.
+  CHECK_EQUAL(0, k_e->current_device_op);
+
+  // And let go.
+  // (Don't check for CL_INVALID_EVENT on a second release of each of
+  // these events because the events might be reused.)
+  CHECK_EQUAL(CL_SUCCESS, clReleaseMemObject(mem));
+  CHECK_EQUAL(CL_SUCCESS, clReleaseEvent(k_e));
+  CHECK_EQUAL(CL_SUCCESS, clReleaseKernel(k));
+
+  // Clean up device global
+  m_device->def.autodiscovery_def.device_global_mem_defs.clear();
+
+  acl_print_debug_msg("DONE!\n");
+}
+
 TEST(acl_kernel_reprogram_scheduler, use_host_buf_as_arg) {
   // Must be able to use a host-side buffer as a kernel argument.
   cl_int status = 0;