[WebGPU EP] Add environment variable to dump shader code to a file, move shader key validation to nightly build

.github/workflows/nightly_webgpu.yml

@@ -0,0 +1,81 @@
+name: Nightly ONNX Runtime WebGPU Builds
+
+on:
+  schedule:
+  - cron: '0 9 * * *'  # Daily at 09:00 UTC
+  workflow_dispatch:
+  # TODO remove the `push` trigger - only for testing
+  push:
+    branches:
+    - edgchen1/webgpu_ep_dump_shader_code
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  webgpu_shader_key_validation:
+    runs-on: [
+      "self-hosted",
+      "1ES.Pool=onnxruntime-github-Win2022-GPU-A10",
+      "JobId=webgpu_shader_validation-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"
+      ]
+    timeout-minutes: 90
+    env:
+      ALLOW_RELEASED_ONNX_OPSET_ONLY: "0"
+      ONNXRUNTIME_TEST_GPU_DEVICE_ID: "0"
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+        submodules: none
+
+    - name: Setup Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+        architecture: x64
+
+    - name: Locate vcvarsall and Setup Env
+      uses: ./.github/actions/locate-vcvarsall-and-setup-env
+      with:
+        architecture: x64
+
+    - name: Install python modules
+      run: python -m pip install -r tools\ci_build\github\windows\python\requirements.txt
+      shell: cmd
+      working-directory: ${{ github.workspace }}
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version: "20.x"
+
+    - name: Build and Test
+      shell: pwsh
+      run: |
+        $env:ORT_WEBGPU_EP_SHADER_DUMP_FILE = "${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log"
+
+        python.exe ${{ github.workspace }}\tools\ci_build\build.py `
+          --config RelWithDebInfo `
+          --build_dir ${{ github.workspace }} `
+          --use_binskim_compliant_compile_flags `
+          --cmake_generator "Visual Studio 17 2022" `
+          --build_shared_lib `
+          --use_webgpu `
+          --wgsl_template static `
+          --cmake_extra_defines onnxruntime_BUILD_DAWN_SHARED_LIBRARY=ON `
+          --update `
+          --build --parallel `
+          --test
+
+    - name: Check log file
+      shell: cmd
+      run: |
+        dir ${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log
+
+    - name: Validate shader keys
+      uses: ./.github/actions/webgpu-validate-shader-key
+      with:
+        log_file_path: ${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log

onnxruntime/core/providers/webgpu/program_manager.cc

@@ -2,9 +2,12 @@
 // Licensed under the MIT License.
 
 #include <algorithm>
+#include <fstream>
+#include <memory>
 
 #include "core/common/common.h"
 #include "core/common/logging/logging.h"
+#include "core/platform/env_var.h"
 
 #include "core/providers/webgpu/program_manager.h"
 #include "core/providers/webgpu/shader_helper.h"
@@ -18,6 +21,17 @@ ProgramArtifact::ProgramArtifact(const ProgramBase& program, wgpu::ComputePipeli
       compute_pipeline{compute_pipeline},
       shape_uniform_ranks{shape_uniform_ranks} {}
 
+ProgramManager::ProgramManager(WebGpuContext& webgpu_context)
+    : webgpu_context_{webgpu_context} {
+  if (std::string dump_file_path = onnxruntime::detail::GetEnvironmentVar("ORT_WEBGPU_EP_SHADER_DUMP_FILE");
+      !dump_file_path.empty()) {
+    auto dump_file = std::make_shared<std::ofstream>(dump_file_path.c_str(), std::ios::app);
+    shader_dump_fn_ = [dump_file = std::move(dump_file)](std::string_view shader_content) {
+      *dump_file << shader_content << "\n";
+    };
+  }
+}
+
 Status ProgramManager::NormalizeDispatchGroupSize(uint32_t& x, uint32_t& y, uint32_t& z) const {
   ORT_RETURN_IF(x == 0 || y == 0 || z == 0, "Invalid dispatch group size (", x, ", ", y, ", ", z, ")");
 
@@ -66,9 +80,7 @@ Status ProgramManager::Build(const ProgramBase& program,
                              const ProgramMetadata& program_metadata,
                              const std::span<uint32_t> inputs_segments,
                              const std::span<uint32_t> outputs_segments,
-#ifndef NDEBUG  // if debug build
                              const std::string& program_key,
-#endif
                              uint32_t normalized_dispatch_x,
                              uint32_t normalized_dispatch_y,
                              uint32_t normalized_dispatch_z,
@@ -100,17 +112,24 @@ Status ProgramManager::Build(const ProgramBase& program,
   std::string code;
   ORT_RETURN_IF_ERROR(shader_helper.GenerateSourceCode(code, shape_uniform_ranks));
 
-  LOGS_DEFAULT(VERBOSE) << "\n=== WebGPU Shader code [" << program.Name()
-#ifndef NDEBUG  // if debug build
-                        << ", Key=\"" << program_key << "\""
-#endif
-                        << "] Start ===\n\n"
-                        << code
-                        << "\n=== WebGPU Shader code [" << program.Name()
-#ifndef NDEBUG  // if debug build
-                        << ", Key=\"" << program_key << "\""
-#endif
-                        << "] End ===\n";
+  // Dump shader code, if requested. It is dumped to `shader_dump_fn_` if set or VERBOSE logging otherwise.
+  {
+    const auto shader_content = [&program, &program_key, &code]() {
+      return MakeString("\n=== WebGPU Shader code [", program.Name(),
+                        ", Key=\"", program_key, "\"",
+                        "] Start ===\n\n",
+                        code,
+                        "\n=== WebGPU Shader code [", program.Name(),
+                        ", Key=\"", program_key, "\"",
+                        "] End ===\n");
+    };
+
+    if (shader_dump_fn_) {
+      shader_dump_fn_(shader_content());
+    } else {
+      LOGS_DEFAULT(VERBOSE) << shader_content();
+    }
+  }
 
   wgpu::ShaderSourceWGSL wgsl_source{};
   wgsl_source.code = code.c_str();

onnxruntime/core/providers/webgpu/program_manager.h

@@ -3,8 +3,10 @@
 
 #pragma once
 
+#include <functional>
 #include <span>
 #include <string>
+#include <string_view>
 #include <unordered_map>
 
 #include "core/providers/webgpu/webgpu_external_header.h"
@@ -36,7 +38,7 @@
 
 class ProgramManager {
  public:
-  ProgramManager(WebGpuContext& webgpu_context) : webgpu_context_(webgpu_context) {}
+  ProgramManager(WebGpuContext& webgpu_context);
 
   Status NormalizeDispatchGroupSize(uint32_t& x, uint32_t& y, uint32_t& z) const;
   Status CalculateSegmentsForInputsAndOutputs(const ProgramBase& program, std::vector<uint32_t>& inputs_segments, std::vector<uint32_t>& outputs_segments) const;
@@ -45,9 +47,7 @@
                const ProgramMetadata& metadata,
                const std::span<uint32_t> inputs_segments,
                const std::span<uint32_t> outputs_segments,
-#ifndef NDEBUG  // if debug build
                const std::string& program_key,
-#endif
                uint32_t normalized_dispatch_x,
                uint32_t normalized_dispatch_y,
                uint32_t normalized_dispatch_z,
@@ -59,6 +59,8 @@
  private:
   std::unordered_map<std::string, ProgramArtifact> programs_;
   WebGpuContext& webgpu_context_;
+
+  std::function<void(std::string_view)> shader_dump_fn_;
 };
 
 }  // namespace webgpu

onnxruntime/core/providers/webgpu/webgpu_context.cc

@@ -303,9 +303,7 @@ Status WebGpuContext::Run(ComputeContextBase& context, const ProgramBase& progra
                                       metadata,
                                       inputs_segments,
                                       outputs_segments,
-#ifndef NDEBUG  // if debug build
                                       key,
-#endif
                                       x,
                                       y,
                                       z,