Skip to content

Commit ec0014c

Browse files
authored
Merge pull request #1074 from intel/syncing_msft_4_5_26
Backmerging Msft commits
2 parents 1a4e22d + 8ff82ba commit ec0014c

119 files changed

Lines changed: 4991 additions & 341 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/linux_cuda_plugin_ci.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ jobs:
3636
--cuda_version=12.8
3737
--cuda_home=/usr/local/cuda-12.8
3838
--cudnn_home=/usr/local/cuda-12.8
39+
--enable_cuda_profiling
3940
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
4041
--cmake_extra_defines onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON
4142
python_path_prefix: 'PATH=/opt/python/cp312-cp312/bin:$PATH'

.github/workflows/windows_cuda_plugin.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ jobs:
8383
--skip_tests `
8484
--use_vcpkg `
8585
--use_vcpkg_ms_internal_asset_cache `
86+
--enable_cuda_profiling `
8687
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 `
8788
--cmake_extra_defines onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON
8889

README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@
2020
- ONNX Runtime Inferencing: [microsoft/onnxruntime-inference-examples](https://github.com/microsoft/onnxruntime-inference-examples)
2121
- ONNX Runtime Training: [microsoft/onnxruntime-training-examples](https://github.com/microsoft/onnxruntime-training-examples)
2222

23+
* **Plugin EP repositories**:
24+
- ONNX Runtime QNN Plugin EP: [onnxruntime/onnxruntime-qnn](https://github.com/onnxruntime/onnxruntime-qnn)
25+
2326
## Releases
2427

2528
The current release and past releases can be found here: https://github.com/microsoft/onnxruntime/releases.

cmake/onnxruntime_providers_cuda_plugin.cmake

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,11 @@ target_link_libraries(onnxruntime_providers_cuda_plugin PRIVATE
265265
${PROTOBUF_LIB}
266266
)
267267

268+
if (onnxruntime_ENABLE_CUDA_PROFILING)
269+
target_link_libraries(onnxruntime_providers_cuda_plugin PRIVATE CUDA::cupti)
270+
target_compile_definitions(onnxruntime_providers_cuda_plugin PRIVATE ENABLE_CUDA_PROFILING)
271+
endif()
272+
268273
# Default plugin EP version to ORT_VERSION with "-dev" suffix if not explicitly provided.
269274
if(NOT DEFINED onnxruntime_PLUGIN_EP_VERSION)
270275
set(onnxruntime_PLUGIN_EP_VERSION "${ORT_VERSION}-dev")

cmake/onnxruntime_unittests.cmake

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1025,7 +1025,8 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
10251025
endif()
10261026
list(REMOVE_ITEM all_tests
10271027
"${TEST_SRC_DIR}/providers/cpu/reduction/reduction_ops_test.cc"
1028-
"${TEST_SRC_DIR}/providers/cpu/tensor/grid_sample_test.cc")
1028+
"${TEST_SRC_DIR}/providers/cpu/tensor/grid_sample_test.cc"
1029+
"${TEST_SRC_DIR}/providers/cpu/tensor/grid_sample_test_custom.cc")
10291030
endif()
10301031

10311032
if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR IOS)

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,42 @@ static NativeMethods()
880880
api_.CreateExternalInitializerInfo,
881881
typeof(DOrtCreateExternalInitializerInfo));
882882

883+
// Version 24 additions
884+
OrtGetNumHardwareDevices =
885+
(DOrtGetNumHardwareDevices)Marshal.GetDelegateForFunctionPointer(
886+
api_.GetNumHardwareDevices,
887+
typeof(DOrtGetNumHardwareDevices));
888+
889+
OrtGetHardwareDevices =
890+
(DOrtGetHardwareDevices)Marshal.GetDelegateForFunctionPointer(
891+
api_.GetHardwareDevices,
892+
typeof(DOrtGetHardwareDevices));
893+
894+
OrtGetHardwareDeviceEpIncompatibilityDetails =
895+
(DOrtGetHardwareDeviceEpIncompatibilityDetails)Marshal.GetDelegateForFunctionPointer(
896+
api_.GetHardwareDeviceEpIncompatibilityDetails,
897+
typeof(DOrtGetHardwareDeviceEpIncompatibilityDetails));
898+
899+
OrtDeviceEpIncompatibilityDetails_GetReasonsBitmask =
900+
(DOrtDeviceEpIncompatibilityDetails_GetReasonsBitmask)Marshal.GetDelegateForFunctionPointer(
901+
api_.DeviceEpIncompatibilityDetails_GetReasonsBitmask,
902+
typeof(DOrtDeviceEpIncompatibilityDetails_GetReasonsBitmask));
903+
904+
OrtDeviceEpIncompatibilityDetails_GetNotes =
905+
(DOrtDeviceEpIncompatibilityDetails_GetNotes)Marshal.GetDelegateForFunctionPointer(
906+
api_.DeviceEpIncompatibilityDetails_GetNotes,
907+
typeof(DOrtDeviceEpIncompatibilityDetails_GetNotes));
908+
909+
OrtDeviceEpIncompatibilityDetails_GetErrorCode =
910+
(DOrtDeviceEpIncompatibilityDetails_GetErrorCode)Marshal.GetDelegateForFunctionPointer(
911+
api_.DeviceEpIncompatibilityDetails_GetErrorCode,
912+
typeof(DOrtDeviceEpIncompatibilityDetails_GetErrorCode));
913+
914+
OrtReleaseDeviceEpIncompatibilityDetails =
915+
(DOrtReleaseDeviceEpIncompatibilityDetails)Marshal.GetDelegateForFunctionPointer(
916+
api_.ReleaseDeviceEpIncompatibilityDetails,
917+
typeof(DOrtReleaseDeviceEpIncompatibilityDetails));
918+
883919
OrtCreateSharedAllocator =
884920
(DOrtCreateSharedAllocator)Marshal.GetDelegateForFunctionPointer(
885921
api_.CreateSharedAllocator,
@@ -2820,6 +2856,75 @@ out IntPtr /* OrtExternalInitializerInfo** */ newExternalInfo
28202856

28212857
#endregion
28222858

2859+
#region Hardware Device EP Compatibility API
2860+
2861+
/// <summary>
2862+
/// Get the number of available hardware devices.
2863+
/// </summary>
2864+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2865+
public delegate IntPtr /* OrtStatus* */ DOrtGetNumHardwareDevices(
2866+
IntPtr /* const OrtEnv* */ env,
2867+
out UIntPtr /* size_t* */ numDevices);
2868+
2869+
/// <summary>
2870+
/// Get the list of available hardware devices.
2871+
/// </summary>
2872+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2873+
public delegate IntPtr /* OrtStatus* */ DOrtGetHardwareDevices(
2874+
IntPtr /* const OrtEnv* */ env,
2875+
[Out] IntPtr[] /* const OrtHardwareDevice** */ devices,
2876+
UIntPtr /* size_t */ numDevices);
2877+
2878+
/// <summary>
2879+
/// Check for known incompatibility issues between hardware device and a specific execution provider.
2880+
/// </summary>
2881+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2882+
public delegate IntPtr /* OrtStatus* */ DOrtGetHardwareDeviceEpIncompatibilityDetails(
2883+
IntPtr /* const OrtEnv* */ env,
2884+
byte[] /* const char* */ epName,
2885+
IntPtr /* const OrtHardwareDevice* */ hw,
2886+
out IntPtr /* OrtDeviceEpIncompatibilityDetails** */ details);
2887+
2888+
/// <summary>
2889+
/// Get the incompatibility reasons bitmask from OrtDeviceEpIncompatibilityDetails.
2890+
/// </summary>
2891+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2892+
public delegate IntPtr /* OrtStatus* */ DOrtDeviceEpIncompatibilityDetails_GetReasonsBitmask(
2893+
IntPtr /* const OrtDeviceEpIncompatibilityDetails* */ details,
2894+
out uint /* uint32_t* */ reasonsBitmask);
2895+
2896+
/// <summary>
2897+
/// Get the notes from OrtDeviceEpIncompatibilityDetails.
2898+
/// </summary>
2899+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2900+
public delegate IntPtr /* OrtStatus* */ DOrtDeviceEpIncompatibilityDetails_GetNotes(
2901+
IntPtr /* const OrtDeviceEpIncompatibilityDetails* */ details,
2902+
out IntPtr /* const char** */ notes);
2903+
2904+
/// <summary>
2905+
/// Get the EP-specific error code from OrtDeviceEpIncompatibilityDetails.
2906+
/// </summary>
2907+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2908+
public delegate IntPtr /* OrtStatus* */ DOrtDeviceEpIncompatibilityDetails_GetErrorCode(
2909+
IntPtr /* const OrtDeviceEpIncompatibilityDetails* */ details,
2910+
out int /* int32_t* */ errorCode);
2911+
2912+
/// <summary>
2913+
/// Release an OrtDeviceEpIncompatibilityDetails instance.
2914+
/// </summary>
2915+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2916+
public delegate void DOrtReleaseDeviceEpIncompatibilityDetails(IntPtr /* OrtDeviceEpIncompatibilityDetails* */ details);
2917+
2918+
public static DOrtGetNumHardwareDevices OrtGetNumHardwareDevices;
2919+
public static DOrtGetHardwareDevices OrtGetHardwareDevices;
2920+
public static DOrtGetHardwareDeviceEpIncompatibilityDetails OrtGetHardwareDeviceEpIncompatibilityDetails;
2921+
public static DOrtDeviceEpIncompatibilityDetails_GetReasonsBitmask OrtDeviceEpIncompatibilityDetails_GetReasonsBitmask;
2922+
public static DOrtDeviceEpIncompatibilityDetails_GetNotes OrtDeviceEpIncompatibilityDetails_GetNotes;
2923+
public static DOrtDeviceEpIncompatibilityDetails_GetErrorCode OrtDeviceEpIncompatibilityDetails_GetErrorCode;
2924+
public static DOrtReleaseDeviceEpIncompatibilityDetails OrtReleaseDeviceEpIncompatibilityDetails;
2925+
2926+
#endregion
2927+
28232928
#region Auto EP API related
28242929
//
28252930
// OrtKeyValuePairs

csharp/src/Microsoft.ML.OnnxRuntime/OrtEnv.shared.cs

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,27 @@ public enum OrtCompiledModelCompatibility
2222
EP_UNSUPPORTED = 3,
2323
}
2424

25+
/// <summary>
26+
/// Reasons why an execution provider might not be compatible with a device.
27+
/// </summary>
28+
/// <remarks>
29+
/// This is a flags enum. Multiple reasons can be combined using bitwise OR.
30+
/// </remarks>
31+
[Flags]
32+
public enum OrtDeviceEpIncompatibilityReason : uint
33+
{
34+
/// <summary>No incompatibility.</summary>
35+
None = 0,
36+
/// <summary>Driver is incompatible with the execution provider.</summary>
37+
DriverIncompatible = 1 << 0,
38+
/// <summary>Device itself is incompatible with the execution provider.</summary>
39+
DeviceIncompatible = 1 << 1,
40+
/// <summary>Required dependency is missing.</summary>
41+
MissingDependency = 1 << 2,
42+
/// <summary>Unknown incompatibility reason.</summary>
43+
Unknown = 1u << 31
44+
}
45+
2546
/// <summary>
2647
/// Delegate for logging function callback.
2748
/// Supply your function and register it with the environment to receive logging callbacks via
@@ -593,6 +614,76 @@ public string GetCompatibilityInfoFromModelBytes(byte[] modelData, string epType
593614
return NativeOnnxValueHelper.StringFromNativeUtf8(compatInfoPtr, allocator);
594615
}
595616

617+
/// <summary>
618+
/// Get the number of available hardware devices.
619+
/// </summary>
620+
/// <returns>The number of hardware devices discovered on the system.</returns>
621+
public int GetNumHardwareDevices()
622+
{
623+
NativeApiStatus.VerifySuccess(
624+
NativeMethods.OrtGetNumHardwareDevices(Handle, out UIntPtr numDevices));
625+
return checked((int)numDevices);
626+
}
627+
628+
/// <summary>
629+
/// Get the list of available hardware devices.
630+
/// </summary>
631+
/// <returns>A list of OrtHardwareDevice objects. The underlying native handles are owned by ORT and should not be released.</returns>
632+
public IReadOnlyList<OrtHardwareDevice> GetHardwareDevices()
633+
{
634+
NativeApiStatus.VerifySuccess(
635+
NativeMethods.OrtGetNumHardwareDevices(Handle, out UIntPtr numDevices));
636+
637+
int count = checked((int)numDevices);
638+
if (count == 0)
639+
{
640+
return Array.Empty<OrtHardwareDevice>();
641+
}
642+
643+
var devicePtrs = new IntPtr[count];
644+
NativeApiStatus.VerifySuccess(
645+
NativeMethods.OrtGetHardwareDevices(Handle, devicePtrs, numDevices));
646+
647+
var devices = new OrtHardwareDevice[count];
648+
for (int i = 0; i < count; i++)
649+
{
650+
devices[i] = new OrtHardwareDevice(devicePtrs[i]);
651+
}
652+
return devices;
653+
}
654+
655+
/// <summary>
656+
/// Check for known incompatibility issues between a hardware device and a specific execution provider.
657+
/// </summary>
658+
/// <param name="epName">The name of the execution provider to check.</param>
659+
/// <param name="hardwareDevice">The hardware device to check for incompatibility.</param>
660+
/// <returns>Details about incompatibility including reasons and notes.</returns>
661+
/// <remarks>
662+
/// This method can be used with built-in execution providers without calling
663+
/// RegisterExecutionProviderLibrary.
664+
/// For execution providers supplied by external libraries, the provider library must be
665+
/// registered before calling this method.
666+
/// If the returned details have non-zero reasons, the device is not compatible.
667+
/// However, zero reasons don't guarantee 100% compatibility for all models.
668+
/// </remarks>
669+
public OrtDeviceEpIncompatibilityDetails GetHardwareDeviceEpIncompatibilityDetails(
670+
string epName, OrtHardwareDevice hardwareDevice)
671+
{
672+
if (epName == null)
673+
throw new ArgumentNullException(nameof(epName));
674+
if (epName.Length == 0)
675+
throw new ArgumentException("epName must be non-empty", nameof(epName));
676+
if (hardwareDevice == null)
677+
throw new ArgumentNullException(nameof(hardwareDevice));
678+
679+
var epNameUtf8 = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(epName);
680+
NativeApiStatus.VerifySuccess(
681+
NativeMethods.OrtGetHardwareDeviceEpIncompatibilityDetails(
682+
Handle, epNameUtf8, hardwareDevice.Handle, out IntPtr details));
683+
684+
return new OrtDeviceEpIncompatibilityDetails(details);
685+
}
686+
596687

597688
/// <summary>
598689
/// Get/Set log level property of OrtEnv instance
@@ -738,4 +829,120 @@ protected override bool ReleaseHandle()
738829
}
739830
#endregion
740831
}
832+
833+
/// <summary>
834+
/// Contains details about why an execution provider is incompatible with a hardware device.
835+
/// </summary>
836+
/// <remarks>
837+
/// This class wraps the native OrtDeviceEpIncompatibilityDetails object.
838+
/// Use the properties to query specific incompatibility information.
839+
/// </remarks>
840+
public sealed class OrtDeviceEpIncompatibilityDetails : IDisposable
841+
{
842+
private IntPtr _handle;
843+
private bool _disposed = false;
844+
845+
/// <summary>
846+
/// Creates a new OrtDeviceEpIncompatibilityDetails wrapper.
847+
/// </summary>
848+
/// <param name="handle">The native handle to wrap.</param>
849+
internal OrtDeviceEpIncompatibilityDetails(IntPtr handle)
850+
{
851+
_handle = handle;
852+
}
853+
854+
/// <summary>
855+
/// Gets the bitmask of incompatibility reasons.
856+
/// </summary>
857+
/// <remarks>
858+
/// If this value is 0 (None), there are no known incompatibility issues.
859+
/// However, this doesn't guarantee 100% compatibility for all models.
860+
/// </remarks>
861+
public OrtDeviceEpIncompatibilityReason ReasonsBitmask
862+
{
863+
get
864+
{
865+
if (_disposed)
866+
throw new ObjectDisposedException(nameof(OrtDeviceEpIncompatibilityDetails));
867+
868+
NativeApiStatus.VerifySuccess(
869+
NativeMethods.OrtDeviceEpIncompatibilityDetails_GetReasonsBitmask(_handle, out uint bitmask));
870+
return (OrtDeviceEpIncompatibilityReason)bitmask;
871+
}
872+
}
873+
874+
/// <summary>
875+
/// Gets human-readable notes about the incompatibility.
876+
/// </summary>
877+
/// <remarks>
878+
/// May be null if no notes are available.
879+
/// </remarks>
880+
public string Notes
881+
{
882+
get
883+
{
884+
if (_disposed)
885+
throw new ObjectDisposedException(nameof(OrtDeviceEpIncompatibilityDetails));
886+
887+
NativeApiStatus.VerifySuccess(
888+
NativeMethods.OrtDeviceEpIncompatibilityDetails_GetNotes(_handle, out IntPtr notesPtr));
889+
890+
if (notesPtr == IntPtr.Zero)
891+
return null;
892+
893+
return NativeOnnxValueHelper.StringFromNativeUtf8(notesPtr);
894+
}
895+
}
896+
897+
/// <summary>
898+
/// Gets the EP-specific error code.
899+
/// </summary>
900+
/// <remarks>
901+
/// This allows Independent Hardware Vendors (IHVs) to define their own error codes
902+
/// to provide additional details about device incompatibility.
903+
/// A value of 0 indicates no error code was set.
904+
/// </remarks>
905+
public int ErrorCode
906+
{
907+
get
908+
{
909+
if (_disposed)
910+
throw new ObjectDisposedException(nameof(OrtDeviceEpIncompatibilityDetails));
911+
912+
NativeApiStatus.VerifySuccess(
913+
NativeMethods.OrtDeviceEpIncompatibilityDetails_GetErrorCode(_handle, out int errorCode));
914+
return errorCode;
915+
}
916+
}
917+
918+
/// <summary>
919+
/// Disposes the native resources.
920+
/// </summary>
921+
public void Dispose()
922+
{
923+
Dispose(true);
924+
GC.SuppressFinalize(this);
925+
}
926+
927+
private void Dispose(bool disposing)
928+
{
929+
if (!_disposed)
930+
{
931+
if (_handle != IntPtr.Zero)
932+
{
933+
NativeMethods.OrtReleaseDeviceEpIncompatibilityDetails(_handle);
934+
_handle = IntPtr.Zero;
935+
}
936+
_disposed = true;
937+
}
938+
}
939+
940+
/// <summary>
941+
/// Finalizer.
942+
/// </summary>
943+
~OrtDeviceEpIncompatibilityDetails()
944+
{
945+
Dispose(false);
946+
}
947+
}
741948
}

0 commit comments

Comments
 (0)