DDI extension support

- Support in the L0 Loader for https://oneapi-src.github.io/level-zero-spec/level-zero/latest/core/EXT_Driver_DDIHandles.html#ze-extension-driver-ddi-handles
- To improve the speed for each call of L0 apis into the correct driver, the new support enables for the driver to allocate a header that stores that driver's ddi tables such that the Loader does not need to perform translation of handles to/from ze_object_t.
- Given a driver supports the new extension, the loader will no longer create the ze_object_t and instead use the packed ddi tables in each handle_t.
- The code has been refactored such that the legacy path will continue to work for the driver that has not converted over without interfering with a driver which now uses the "fast" path.
- This new code path also removes the need for handle translation by the user if the drivers all support the new extension.

Signed-off-by: Neil R. Spruit <neil.r.spruit@intel.com>

Author: nrspruit

samples/zello_world/zello_world.cpp

@@ -131,12 +131,9 @@ int main( int argc, char *argv[] )
 
         for( uint32_t driver = 0; driver < driverCount; ++driver )
         {
+            std::cout << "Driver # " << driver << "\n";
             pDriver = drivers[driver];
             pDevice = findDevice( pDriver, type );
-            if( pDevice )
-            {
-                break;
-            }
         }
     }
 

scripts/generate_code.py

@@ -141,6 +141,23 @@ def _mako_loader_cpp(path, namespace, tags, version, specs, meta):
     filename = "%s.cpp"%(name)
     fout = os.path.join(path, filename)
 
+    print("Generating %s..."%fout)
+    loc += util.makoWrite(
+        fin, fout,
+        name=name,
+        ver=version,
+        namespace=namespace,
+        tags=tags,
+        specs=specs,
+        meta=meta)
+
+    template = "ldrddi_legacy.cpp.mako"
+    fin = os.path.join(templates_dir, template)
+
+    name = "%s_ldrddi_legacy"%(namespace)
+    filename = "%s.cpp"%(name)
+    fout = os.path.join(path, filename)
+
     print("Generating %s..."%fout)
     loc += util.makoWrite(
         fin, fout,

scripts/templates/helper.py

@@ -1702,6 +1702,43 @@ def get_class_function_objs(specs, cname, minVersion = 0, maxVersion = 9999):
                     objects.append(obj)
     return sorted(objects, key=lambda obj: (float(obj.get('version',"1.0"))*10000) + int(obj.get('ordinal',"100")))
 
+"""
+Public:
+    returns the version of a function object
+"""
+def get_version(obj):
+    if obj_traits.is_function(obj):
+        ret_version = "ZE_API_VERSION_FORCE_UINT32"
+        version = obj.get('version')
+        if version == "1.0":
+            ret_version = "ZE_API_VERSION_1_0"
+        if version == "1.1":
+            ret_version = "ZE_API_VERSION_1_1"
+        if version == "1.2":
+            ret_version = "ZE_API_VERSION_1_2"
+        if version == "1.3":
+            ret_version = "ZE_API_VERSION_1_3"
+        if version == "1.4":
+            ret_version = "ZE_API_VERSION_1_4"
+        if version == "1.5":
+            ret_version = "ZE_API_VERSION_1_5"
+        if version == "1.6":
+            ret_version = "ZE_API_VERSION_1_6"
+        if version == "1.7":
+            ret_version = "ZE_API_VERSION_1_7"
+        if version == "1.8":
+            ret_version = "ZE_API_VERSION_1_8"
+        if version == "1.9":
+            ret_version = "ZE_API_VERSION_1_9"
+        if version == "1.10":
+            ret_version = "ZE_API_VERSION_1_10"
+        if version == "1.11":
+            ret_version = "ZE_API_VERSION_1_11"
+        if version == "1.12":
+            ret_version = "ZE_API_VERSION_1_12"
+        assert(ret_version != "ZE_API_VERSION_FORCE_UINT32")
+        return ret_version
+
 """
 Public:
     returns a list of all non-experimental function objs and a list of experimental function objs for the specified class

scripts/templates/ldrddi.cpp.mako

@@ -161,8 +161,28 @@ namespace loader
                 {
                     for( uint32_t i = 0; i < library_driver_handle_count; ++i ) {
                         uint32_t driver_index = total_driver_handle_count + i;
-                        ${obj['params'][1]['name']}[ driver_index ] = reinterpret_cast<${n}_driver_handle_t>(
-                            context->${n}_driver_factory.getInstance( ${obj['params'][1]['name']}[ driver_index ], &drv.dditable ) );
+                        if (drv.driverDDIHandleSupportQueried == false) {
+                            drv.properties = {};
+                            drv.properties.stype = ZE_STRUCTURE_TYPE_DRIVER_DDI_HANDLES_EXT_PROPERTIES;
+                            drv.properties.pNext = nullptr;
+                            ze_driver_properties_t driverProperties = {};
+                            driverProperties.stype = ZE_STRUCTURE_TYPE_DRIVER_PROPERTIES;
+                            driverProperties.pNext = nullptr;
+                            driverProperties.pNext = &drv.properties;
+                            ze_result_t res = drv.dditable.ze.Driver.pfnGetProperties(${obj['params'][1]['name']}[ driver_index ], &driverProperties);
+                            if (res != ZE_RESULT_SUCCESS) {
+                                if (loader::context->debugTraceEnabled) {
+                                    std::string message = drv.name + " failed zeDriverGetProperties query, returned ";
+                                    loader::context->debug_trace_message(message, loader::to_string(res));
+                                }
+                                return res;
+                            }
+                            drv.driverDDIHandleSupportQueried = true;
+                        }
+                        if (!(drv.properties.flags & ZE_DRIVER_DDI_HANDLE_EXT_FLAG_DDI_HANDLE_EXT_SUPPORTED)) {
+                            ${obj['params'][1]['name']}[ driver_index ] = reinterpret_cast<${n}_driver_handle_t>(
+                                context->${n}_driver_factory.getInstance( ${obj['params'][1]['name']}[ driver_index ], &drv.dditable ) );
+                        }
                     }
                 }
                 catch( std::bad_alloc& )
@@ -184,167 +204,62 @@ namespace loader
         %else:
         %for i, item in enumerate(th.get_loader_prologue(n, tags, obj, meta)):
         %if 0 == i:
-        // extract driver's function pointer table
+        // extract handle's function pointer table
         %if 'range' in item:
-        auto dditable = reinterpret_cast<${item['obj']}*>( ${item['name']}[ 0 ] )->dditable;
-        %else:
-        auto dditable = reinterpret_cast<${item['obj']}*>( ${item['name']} )->dditable;
-        %endif
-        auto ${th.make_pfn_name(n, tags, obj)} = dditable->${n}.${th.get_table_name(n, tags, obj)}.${th.make_pfn_name(n, tags, obj)};
-        if( nullptr == ${th.make_pfn_name(n, tags, obj)} )
-            return ${X}_RESULT_ERROR_UNINITIALIZED;
-
+        %if namespace == "ze":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']}[ 0 ] )->pCore;
+        %elif namespace == "zet":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']}[ 0 ] )->pTools;
+        %elif namespace == "zes":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']}[ 0 ] )->pSysman;
         %endif
-        %if 'range' in item:
-        <%
-        add_local = True%>// convert loader handles to driver handles
-        auto ${item['name']}Local = new ${item['type']} [${item['range'][1]}];
-        <%
-        arrays_to_delete.append(item['name']+ 'Local')
-        %>for( size_t i = ${item['range'][0]}; ( nullptr != ${item['name']} ) && ( i < ${item['range'][1]} ); ++i )
-            ${item['name']}Local[ i ] = reinterpret_cast<${item['obj']}*>( ${item['name']}[ i ] )->handle;
-        %else:
-        %if item['optional']:
-        // convert loader handle to driver handle
-        ${item['name']} = ( ${item['name']} ) ? reinterpret_cast<${item['obj']}*>( ${item['name']} )->handle : nullptr;
         %else:
-        %if re.match(r"\w+ImageDestroy$", th.make_func_name(n, tags, obj)):
-        // remove the handle from the kernel arugment map
-        {
-            std::lock_guard<std::mutex> lock(context->image_handle_map_lock);
-            context->image_handle_map.erase(reinterpret_cast<ze_image_object_t*>(hImage));
-        }
-        %endif
-        %if re.match(r"\w+SamplerDestroy$", th.make_func_name(n, tags, obj)):
-        // remove the handle from the kernel arugment map
-        {
-            std::lock_guard<std::mutex> lock(context->sampler_handle_map_lock);
-            context->sampler_handle_map.erase(reinterpret_cast<ze_sampler_object_t*>(hSampler));
-        }
-        %endif
-        // convert loader handle to driver handle
-        ${item['name']} = reinterpret_cast<${item['obj']}*>( ${item['name']} )->handle;
+        %if namespace == "ze":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']} )->pCore;
+        %elif namespace == "zet":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']} )->pTools;
+        %elif namespace == "zes":
+        auto dditable = reinterpret_cast<ze_handle_t*>( ${item['name']} )->pSysman;
         %endif
         %endif
-
-        %endfor
-        %if re.match(r"\w+KernelSetArgumentValue$", th.make_func_name(n, tags, obj)):
-        // convert pArgValue to correct handle if applicable
-        void *internalArgValue = const_cast<void *>(pArgValue);
-        if (pArgValue) {
-            // check if the arg value is a translated handle
-            ze_image_object_t **imageHandle = static_cast<ze_image_object_t **>(internalArgValue);
-            ze_sampler_object_t **samplerHandle = static_cast<ze_sampler_object_t **>(internalArgValue);
-            {
-                std::lock_guard<std::mutex> image_lock(context->image_handle_map_lock);
-                std::lock_guard<std::mutex> sampler_lock(context->sampler_handle_map_lock);
-                if( context->image_handle_map.find(*imageHandle) != context->image_handle_map.end() ) {
-                    internalArgValue = &context->image_handle_map[*imageHandle];
-                } else if( context->sampler_handle_map.find(*samplerHandle) != context->sampler_handle_map.end() ) {
-                    internalArgValue = &context->sampler_handle_map[*samplerHandle];
-                }
-            }
+        if (dditable->isValidFlag == 0)
+            return ${X}_RESULT_ERROR_UNINITIALIZED;
+        // Check that api version in the driver is supported by this version of the API
+        if (dditable->version < ${th.get_version(obj)}) {
+            return ${X}_RESULT_ERROR_UNSUPPORTED_VERSION;
         }
+        auto ${th.make_pfn_name(n, tags, obj)} = dditable->${th.get_table_name(n, tags, obj)}->${th.make_pfn_name(n, tags, obj)};
+        if( nullptr == ${th.make_pfn_name(n, tags, obj)} )
+            return ${X}_RESULT_ERROR_UNINITIALIZED;
         %endif
-        ## Workaround due to incorrect defintion of phWaitEvents in the ze headers which missed the range values.
-        ## To be removed once the headers have been updated in a new spec release.
-        %if re.match(r"\w+CommandListAppendMetricQueryEnd$", th.make_func_name(n, tags, obj)):
-        // convert loader handles to driver handles
-        auto phWaitEventsLocal = new ze_event_handle_t [numWaitEvents];
-        for( size_t i = 0; ( nullptr != phWaitEvents ) && ( i < numWaitEvents ); ++i )
-            phWaitEventsLocal[ i ] = reinterpret_cast<ze_event_object_t*>( phWaitEvents[ i ] )->handle;
-
-        // forward to device-driver
-        result = pfnAppendMetricQueryEnd( hCommandList, hMetricQuery, hSignalEvent, numWaitEvents, phWaitEventsLocal );
-        delete []phWaitEventsLocal;
-        %else:
-        // forward to device-driver
-        %if add_local:
-        result = ${th.make_pfn_name(n, tags, obj)}( ${", ".join(th.make_param_lines(n, tags, obj, format=["name", "local"]))} );
-        %for array_name in arrays_to_delete:
-        delete []${array_name};
         %endfor
-        %else:
-        %if re.match(r"\w+KernelSetArgumentValue$", th.make_func_name(n, tags, obj)):
-        result = pfnSetArgumentValue( hKernel, argIndex, argSize, const_cast<const void *>(internalArgValue) );
-        %else:
+        // forward to device-driver
         result = ${th.make_pfn_name(n, tags, obj)}( ${", ".join(th.make_param_lines(n, tags, obj, format=["name"]))} );
-        %endif
-        %endif
-        %endif
-<%
-        del arrays_to_delete
-        del add_local%>
-        %for i, item in enumerate(th.get_loader_epilogue(n, tags, obj, meta)):
-        %if 0 == i:
-        %if not re.match(r"\w+ModuleDynamicLink$", th.make_func_name(n, tags, obj)) and not re.match(r"\w+ModuleCreate$", th.make_func_name(n, tags, obj)):
-        if( ${X}_RESULT_SUCCESS != result )
-            return result;
-
-        %endif
-        %endif
-        %if item['release']:
-        // release loader handle
-        context->${item['factory']}.release( ${item['name']} );
-        %else:
-        try
-        {
-            %if 'range' in item:
-            // convert driver handles to loader handles
-            for( size_t i = ${item['range'][0]}; ( nullptr != ${item['name']} ) && ( i < ${item['range'][1]} ); ++i )
-                ${item['name']}[ i ] = reinterpret_cast<${item['type']}>(
-                    context->${item['factory']}.getInstance( ${item['name']}[ i ], dditable ) );
-            %else:
-            // convert driver handle to loader handle
-            %if item['optional']:
-            if( nullptr != ${item['name']} )
-                *${item['name']} = reinterpret_cast<${item['type']}>(
-                    context->${item['factory']}.getInstance( *${item['name']}, dditable ) );
-            %else:
-            %if re.match(r"\w+ImageCreate$", th.make_func_name(n, tags, obj)) or re.match(r"\w+SamplerCreate$", th.make_func_name(n, tags, obj)) or re.match(r"\w+ImageViewCreateExp$", th.make_func_name(n, tags, obj)):
-            ${item['type']} internalHandlePtr = *${item['name']};
-            %endif
-            *${item['name']} = reinterpret_cast<${item['type']}>(
-                context->${item['factory']}.getInstance( *${item['name']}, dditable ) );
-            %if re.match(r"\w+ImageCreate$", th.make_func_name(n, tags, obj)) or re.match(r"\w+ImageViewCreateExp$", th.make_func_name(n, tags, obj)):
-            // convert loader handle to driver handle and store in map
-            {
-                std::lock_guard<std::mutex> lock(context->image_handle_map_lock);
-                context->image_handle_map.insert({context->ze_image_factory.getInstance( internalHandlePtr, dditable ), internalHandlePtr});
-            }
-            %endif
-            %if re.match(r"\w+SamplerCreate$", th.make_func_name(n, tags, obj)):
-            // convert loader handle to driver handle and store in map
-            {
-                std::lock_guard<std::mutex> lock(context->sampler_handle_map_lock);
-                context->sampler_handle_map.insert({context->ze_sampler_factory.getInstance( internalHandlePtr, dditable ), internalHandlePtr});
-            }
-            %endif
-            %endif
-            %endif
-        }
-        catch( std::bad_alloc& )
-        {
-            result = ${X}_RESULT_ERROR_OUT_OF_HOST_MEMORY;
-        }
-        %endif
-
-        %if 0 == i:
-        %if re.match(r"\w+ModuleDynamicLink$", th.make_func_name(n, tags, obj)) or re.match(r"\w+ModuleCreate$", th.make_func_name(n, tags, obj)):
-        if( ${X}_RESULT_SUCCESS != result )
-            return result;
-
-        %endif
-        %endif
-        %endfor
-        %endif
         return result;
     }
     %if 'condition' in obj:
     #endif // ${th.subt(n, tags, obj['condition'])}
     %endif
 
+    %endif
+    %if re.match(r"Init", obj['name']) or re.match(r"\w+InitDrivers$", th.make_func_name(n, tags, obj)) or re.match(r"\w+DriverGet$", th.make_func_name(n, tags, obj)):
+        return result;
+    }
+    %endif
     %endfor
+
+    ///////////////////////////////////////////////////////////////////////////////
+    /// @brief function for removing the ddi driver tables for ${n}
+    __${x}dlllocal void ${X}_APICALL
+    ${n}DestroyDDiDriverTables(${n}_dditable_driver_t* pDdiTable)
+    {
+        // Delete ddi tables
+%for tbl in th.get_pfntables(specs, meta, n, tags):
+        delete pDdiTable->${tbl['name']};
+%endfor
+        delete pDdiTable;
+    }
+
 } // namespace loader
 
 #if defined(__cplusplus)
@@ -442,6 +357,13 @@ ${tbl['export']['name']}(
         %endif
         {
             // return pointers to loader's DDIs
+            %if namespace == "ze":
+            loader::loaderDispatch->pCore->${tbl['name']} = new ${tbl['type']};
+            %elif namespace == "zet":
+            loader::loaderDispatch->pTools->${tbl['name']} = new ${tbl['type']};
+            %elif namespace == "zes":
+            loader::loaderDispatch->pSysman->${tbl['name']} = new ${tbl['type']};
+            %endif
             %for obj in tbl['functions']:
             %if 'condition' in obj:
         #if ${th.subt(n, tags, obj['condition'])}
@@ -453,6 +375,7 @@ ${tbl['export']['name']}(
         #endif
             %endif
             %endfor
+            ${tbl['export']['name']}Legacy();
         }
         else
         {
@@ -500,4 +423,4 @@ ${tbl['export']['name']}(
 
 #if defined(__cplusplus)
 };
-#endif
+#endif

scripts/templates/ldrddi.h.mako

@@ -32,4 +32,19 @@ namespace loader
 
     %endif
     %endfor
+    __${x}dlllocal void ${X}_APICALL
+    ${n}DestroyDDiDriverTables(${n}_dditable_driver_t* pDdiTable);
 }
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+%for tbl in th.get_pfntables(specs, meta, n, tags):
+__${x}dlllocal void ${X}_APICALL
+${tbl['export']['name']}Legacy();
+%endfor
+
+#if defined(__cplusplus)
+};
+#endif