kernels: use Metadata from kernels-data

This changeset combines a few related changes:

- Use `Metadata` from `kernels-data` to ensure using the same metadata
  data structures in kernel-builder and kernels.
- Make metadata mandatory and make several metadata fields mandatory,
  namely `id`, `version`, `license` the new `name` field.
- Update the `build.toml` format to v4, making the required fields
  mandatory as well as the `torch-noarch` section for torch-noarch
  kernels.
- Remove support for the very old v1 and v2 `build.toml` formats.

Author: danieldk

docs/source/api/kernels.md

@@ -27,3 +27,13 @@
 ### get_locked_kernel
 
 [[autodoc]] kernels.get_locked_kernel
+
+## Classes
+
+### LoadedKernel
+
+[[autodoc]] kernels.LoadedKernel
+
+### RepoInfo
+
+[[autodoc]] kernels.RepoInfo

docs/source/kernel-requirements.md

@@ -42,7 +42,11 @@ metadata. Currently the following top-level keys are supported:
 - `id` (`str`, required): a unique identifier for the kernel. This
   identifier must also be a valid Python module name. If the kernel
   registers Torch ops, they must be registered as `torch.ops.<id>`
+- `name` (`str`, required): then name of the kernel. Replacing dashes
+  by underscores should result in the module name of the kernel.
 - `version` (`int`, required): the kernel version number.
+- `license` (`str`, required): the kernel license in. Refer to the
+  list of [supported license identifiers](https://huggingface.co/docs/hub/repositories-licenses).
 - `backend` (`dict`, required): information about the compute backend that
   this build variant supports.
 - `python-depends` (`list[str]`, optional): list of Python dependencies
@@ -52,9 +56,11 @@ Example `metadata.json`:
 
 ```json
 {
-  "id": "_mykernel_cuda_be238e4",
-  "python-depends": ["einops"],
+  "name": "mykernel",
+  "id": "_mykernel_cuda_7a4e5a7",
   "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": ["einops"],
   "backend": {
     "type": "cuda",
     "archs": ["7.0", "7.2", "7.5", "8.0", "8.6", "8.7", "8.9", "9.0+PTX"]

examples/kernels/cutlass-gemm-tvm-ffi/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "cutlass-gemm-tvm-ffi"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cuda",
     "xpu",
@@ -9,20 +11,20 @@ backends = [
 repo-id = "kernels-test/cutlass-gemm-tvm-ffi"
 
 [tvm-ffi]
-src = [
-    "tvm-ffi-ext/tvm_ffi_binding.cpp",
-]
+src = ["tvm-ffi-ext/tvm_ffi_binding.cpp"]
 
 [kernel.gemm]
 backend = "cuda"
-depends = [
-    "cutlass_3_6",
+depends = ["cutlass_3_6"]
+src = [
+    "gemm.cu",
+    "util.hh",
 ]
-src = ["gemm.cu", "util.hh"]
 
 [kernel.gemm_xpu]
 backend = "xpu"
-depends = [
-    "sycl_tla",
+depends = ["sycl_tla"]
+src = [
+    "gemm_sycl.cpp",
+    "util.hh",
 ]
-src = ["gemm_sycl.cpp", "util.hh"]

examples/kernels/cutlass-gemm/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "cutlass-gemm"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cuda",
     "xpu",
@@ -14,18 +16,18 @@ src = [
     "torch-ext/torch_binding.h",
 ]
 
-[kernel.gemm]
-backend = "cuda"
-depends = [
-    "torch",
-    "cutlass_3_6",
-]
-src = ["gemm.cu"]
-
 [kernel.gemm_xpu]
 backend = "xpu"
 depends = [
     "torch",
     "sycl_tla",
 ]
 src = ["gemm_sycl.cpp"]
+
+[kernel.gemm]
+backend = "cuda"
+depends = [
+    "torch",
+    "cutlass_3_6",
+]
+src = ["gemm.cu"]

examples/kernels/extra-data/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "extra-data"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cpu",
     "cuda",
@@ -12,28 +14,18 @@ backends = [
 repo-id = "kernels-test/extra-data"
 
 [torch]
+pyext = [
+    "json",
+    "py",
+]
 src = [
     "torch-ext/torch_binding.cpp",
     "torch-ext/torch_binding.h",
 ]
-pyext = ["json", "py"]
-
-[kernel.relu]
-backend = "cuda"
-depends = ["torch"]
-src = ["relu_cuda/relu.cu"]
-
-[kernel.relu_metal]
-backend = "metal"
-src = [
-  "relu_metal/relu.mm",
-  "relu_metal/relu.metal",
-  "relu_metal/common.h",
-]
-depends = [ "torch" ]
 
 [kernel.relu_rocm]
 backend = "rocm"
+depends = ["torch"]
 rocm-archs = [
     "gfx906",
     "gfx908",
@@ -45,14 +37,27 @@ rocm-archs = [
     "gfx1100",
     "gfx1101",
 ]
-depends = ["torch"]
 src = ["relu_cuda/relu.cu"]
 
 [kernel.relu_xpu]
 backend = "xpu"
 depends = ["torch"]
 src = ["relu_xpu/relu.cpp"]
 
+[kernel.relu_metal]
+backend = "metal"
+depends = ["torch"]
+src = [
+    "relu_metal/relu.mm",
+    "relu_metal/relu.metal",
+    "relu_metal/common.h",
+]
+
+[kernel.relu]
+backend = "cuda"
+depends = ["torch"]
+src = ["relu_cuda/relu.cu"]
+
 [kernel.relu_cpu]
 backend = "cpu"
 depends = ["torch"]

examples/kernels/relu-backprop-compile/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "relu-backprop-compile"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cuda",
     "rocm",
@@ -21,6 +23,7 @@ src = ["relu_cuda/relu.cu"]
 
 [kernel.relu_rocm]
 backend = "rocm"
+depends = ["torch"]
 rocm-archs = [
     "gfx906",
     "gfx908",
@@ -32,5 +35,4 @@ rocm-archs = [
     "gfx1100",
     "gfx1101",
 ]
-depends = ["torch"]
 src = ["relu_cuda/relu.cu"]

examples/kernels/relu-compiler-flags/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "relu-compiler-flags"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cuda",
     "rocm",
@@ -10,16 +12,26 @@ backends = [
 repo-id = "kernels-test/relu-compiler-flags"
 
 [torch]
-src = ["torch-ext/torch_binding.cpp", "torch-ext/torch_binding.h"]
+src = [
+    "torch-ext/torch_binding.cpp",
+    "torch-ext/torch_binding.h",
+]
+
+[kernel.activation_xpu]
+backend = "xpu"
+depends = ["torch"]
+sycl-flags = ["-DWHO_AM_I_IF_NOT_THE_CANARY"]
+src = ["relu_xpu/relu.cpp"]
 
 [kernel.activation]
 backend = "cuda"
+cuda-flags = ["-DWHO_AM_I_IF_NOT_THE_CANARY"]
 depends = ["torch"]
 src = ["relu_cuda/relu.cu"]
-cuda-flags = ["-DWHO_AM_I_IF_NOT_THE_CANARY"]
 
 [kernel.activation_rocm]
 backend = "rocm"
+depends = ["torch"]
 rocm-archs = [
     "gfx906",
     "gfx908",
@@ -31,12 +43,5 @@ rocm-archs = [
     "gfx1100",
     "gfx1101",
 ]
-depends = ["torch"]
-src = ["relu_cuda/relu.cu"]
 hip-flags = ["-DWHO_AM_I_IF_NOT_THE_CANARY"]
-
-[kernel.activation_xpu]
-backend = "xpu"
-depends = ["torch"]
-src = ["relu_xpu/relu.cpp"]
-sycl-flags = ["-DWHO_AM_I_IF_NOT_THE_CANARY"]
+src = ["relu_cuda/relu.cu"]

examples/kernels/relu-metal-cpp/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "relu"
+version = 1
+license = "Apache-2.0"
 backends = ["metal"]
 
 [general.hub]
@@ -11,13 +13,15 @@ src = [
     "torch-ext/torch_binding.h",
 ]
 
-
 [kernel.relu_metal]
 backend = "metal"
+depends = [
+    "torch",
+    "metal-cpp",
+]
 src = [
-  "relu/relu.cpp",
-  "relu/metallib_loader.mm",
-  "relu/relu_cpp.metal",
-  "relu/common.h",
+    "relu/relu.cpp",
+    "relu/metallib_loader.mm",
+    "relu/relu_cpp.metal",
+    "relu/common.h",
 ]
-depends = [ "torch", "metal-cpp" ]

examples/kernels/relu-nki/build.toml

@@ -1,12 +1,15 @@
 [general]
 name = "relu-nki"
 version = 1
-backends = [
-    "neuron",
-]
+license = "Apache-2.0"
+backends = ["neuron"]
 
 [general.hub]
 repo-id = "kernels-test/relu-nki"
 
 [general.neuron]
 python-depends = ["nki"]
+
+[torch-noarch]
+
+[kernel]

examples/kernels/relu-specific-torch/build.toml

@@ -1,5 +1,7 @@
 [general]
 name = "relu-specific-torch"
+version = 1
+license = "Apache-2.0"
 backends = [
     "cuda",
     "rocm",
@@ -21,6 +23,7 @@ src = ["relu_cuda/relu.cu"]
 
 [kernel.relu_rocm]
 backend = "rocm"
+depends = ["torch"]
 rocm-archs = [
     "gfx906",
     "gfx908",
@@ -32,5 +35,4 @@ rocm-archs = [
     "gfx1100",
     "gfx1101",
 ]
-depends = ["torch"]
 src = ["relu_cuda/relu.cu"]