diff --git a/.github/workflows/nightly_webgpu.yml b/.github/workflows/nightly_webgpu.yml new file mode 100644 index 0000000000000..243ee6436fe0e --- /dev/null +++ b/.github/workflows/nightly_webgpu.yml @@ -0,0 +1,81 @@ +name: Nightly ONNX Runtime WebGPU Builds + +on: + schedule: + - cron: '0 9 * * *' # Daily at 09:00 UTC + workflow_dispatch: + # TODO remove the `push` trigger - only for testing + push: + branches: + - edgchen1/webgpu_ep_dump_shader_code + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + webgpu_shader_key_validation: + runs-on: [ + "self-hosted", + "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", + "JobId=webgpu_shader_validation-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + ] + timeout-minutes: 90 + env: + ALLOW_RELEASED_ONNX_OPSET_ONLY: "0" + ONNXRUNTIME_TEST_GPU_DEVICE_ID: "0" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: none + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + architecture: x64 + + - name: Locate vcvarsall and Setup Env + uses: ./.github/actions/locate-vcvarsall-and-setup-env + with: + architecture: x64 + + - name: Install python modules + run: python -m pip install -r tools\ci_build\github\windows\python\requirements.txt + shell: cmd + working-directory: ${{ github.workspace }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + + - name: Build and Test + shell: pwsh + run: | + $env:ORT_WEBGPU_EP_SHADER_DUMP_FILE = "${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log" + + python.exe ${{ github.workspace }}\tools\ci_build\build.py ` + --config RelWithDebInfo ` + --build_dir ${{ github.workspace }} ` + --use_binskim_compliant_compile_flags ` + --cmake_generator "Visual Studio 17 2022" ` + --build_shared_lib ` + --use_webgpu ` + --wgsl_template static ` + --cmake_extra_defines onnxruntime_BUILD_DAWN_SHARED_LIBRARY=ON ` + --update ` + --build --parallel ` + --test + + - name: Check log file + shell: cmd + run: | + dir ${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log + + - name: Validate shader keys + uses: ./.github/actions/webgpu-validate-shader-key + with: + log_file_path: ${{ github.workspace }}\RelWithDebInfo\RelWithDebInfo\shader_dump.log diff --git a/onnxruntime/core/providers/webgpu/program_manager.cc b/onnxruntime/core/providers/webgpu/program_manager.cc index e4376476a885d..136e7d503f59f 100644 --- a/onnxruntime/core/providers/webgpu/program_manager.cc +++ b/onnxruntime/core/providers/webgpu/program_manager.cc @@ -2,9 +2,12 @@ // Licensed under the MIT License. #include +#include +#include #include "core/common/common.h" #include "core/common/logging/logging.h" +#include "core/platform/env_var.h" #include "core/providers/webgpu/program_manager.h" #include "core/providers/webgpu/shader_helper.h" @@ -18,6 +21,17 @@ ProgramArtifact::ProgramArtifact(const ProgramBase& program, wgpu::ComputePipeli compute_pipeline{compute_pipeline}, shape_uniform_ranks{shape_uniform_ranks} {} +ProgramManager::ProgramManager(WebGpuContext& webgpu_context) + : webgpu_context_{webgpu_context} { + if (std::string dump_file_path = onnxruntime::detail::GetEnvironmentVar("ORT_WEBGPU_EP_SHADER_DUMP_FILE"); + !dump_file_path.empty()) { + auto dump_file = std::make_shared(dump_file_path.c_str(), std::ios::app); + shader_dump_fn_ = [dump_file = std::move(dump_file)](std::string_view shader_content) { + *dump_file << shader_content << "\n"; + }; + } +} + Status ProgramManager::NormalizeDispatchGroupSize(uint32_t& x, uint32_t& y, uint32_t& z) const { ORT_RETURN_IF(x == 0 || y == 0 || z == 0, "Invalid dispatch group size (", x, ", ", y, ", ", z, ")"); @@ -66,9 +80,7 @@ Status ProgramManager::Build(const ProgramBase& program, const ProgramMetadata& program_metadata, const std::span inputs_segments, const std::span outputs_segments, -#ifndef NDEBUG // if debug build const std::string& program_key, -#endif uint32_t normalized_dispatch_x, uint32_t normalized_dispatch_y, uint32_t normalized_dispatch_z, @@ -100,17 +112,24 @@ Status ProgramManager::Build(const ProgramBase& program, std::string code; ORT_RETURN_IF_ERROR(shader_helper.GenerateSourceCode(code, shape_uniform_ranks)); - LOGS_DEFAULT(VERBOSE) << "\n=== WebGPU Shader code [" << program.Name() -#ifndef NDEBUG // if debug build - << ", Key=\"" << program_key << "\"" -#endif - << "] Start ===\n\n" - << code - << "\n=== WebGPU Shader code [" << program.Name() -#ifndef NDEBUG // if debug build - << ", Key=\"" << program_key << "\"" -#endif - << "] End ===\n"; + // Dump shader code, if requested. It is dumped to `shader_dump_fn_` if set or VERBOSE logging otherwise. + { + const auto shader_content = [&program, &program_key, &code]() { + return MakeString("\n=== WebGPU Shader code [", program.Name(), + ", Key=\"", program_key, "\"", + "] Start ===\n\n", + code, + "\n=== WebGPU Shader code [", program.Name(), + ", Key=\"", program_key, "\"", + "] End ===\n"); + }; + + if (shader_dump_fn_) { + shader_dump_fn_(shader_content()); + } else { + LOGS_DEFAULT(VERBOSE) << shader_content(); + } + } wgpu::ShaderSourceWGSL wgsl_source{}; wgsl_source.code = code.c_str(); diff --git a/onnxruntime/core/providers/webgpu/program_manager.h b/onnxruntime/core/providers/webgpu/program_manager.h index 5c4f76d0b4168..afdffe94ea30a 100644 --- a/onnxruntime/core/providers/webgpu/program_manager.h +++ b/onnxruntime/core/providers/webgpu/program_manager.h @@ -3,8 +3,10 @@ #pragma once +#include #include #include +#include #include #include "core/providers/webgpu/webgpu_external_header.h" @@ -36,7 +38,7 @@ class ProgramArtifact { class ProgramManager { public: - ProgramManager(WebGpuContext& webgpu_context) : webgpu_context_(webgpu_context) {} + ProgramManager(WebGpuContext& webgpu_context); Status NormalizeDispatchGroupSize(uint32_t& x, uint32_t& y, uint32_t& z) const; Status CalculateSegmentsForInputsAndOutputs(const ProgramBase& program, std::vector& inputs_segments, std::vector& outputs_segments) const; @@ -45,9 +47,7 @@ class ProgramManager { const ProgramMetadata& metadata, const std::span inputs_segments, const std::span outputs_segments, -#ifndef NDEBUG // if debug build const std::string& program_key, -#endif uint32_t normalized_dispatch_x, uint32_t normalized_dispatch_y, uint32_t normalized_dispatch_z, @@ -59,6 +59,8 @@ class ProgramManager { private: std::unordered_map programs_; WebGpuContext& webgpu_context_; + + std::function shader_dump_fn_; }; } // namespace webgpu diff --git a/onnxruntime/core/providers/webgpu/webgpu_context.cc b/onnxruntime/core/providers/webgpu/webgpu_context.cc index ada9a2e8ab692..c7750198ceebc 100644 --- a/onnxruntime/core/providers/webgpu/webgpu_context.cc +++ b/onnxruntime/core/providers/webgpu/webgpu_context.cc @@ -303,9 +303,7 @@ Status WebGpuContext::Run(ComputeContextBase& context, const ProgramBase& progra metadata, inputs_segments, outputs_segments, -#ifndef NDEBUG // if debug build key, -#endif x, y, z,