Merge branch 'main' into swiglu_offset

Author: vthumbe1503

3rdparty/cudnn-frontend

@@ -6,7 +6,7 @@
 - `CUDA_DEVICE_MAX_CONNECTIONS=1` must be enabled in the environment.
 - For best performance, point-to-point communication via _CUDA Multicast_ needs CUDA Toolkit 12.0+
   and CUDA driver 535+ on devices with compute capability 9.0 or newer.
-- Devices older than compute capability 9.0 require `UB_SKIPMC=1` in the environment in order fall
+- Devices older than compute capability 9.0 require `UB_SKIPMC=1` in the environment in order to fall
   back on a less performant implementation based on CUDA Inter-Process Communication (IPC) handles.
 
 ## Examples
@@ -22,7 +22,7 @@ $ torchrun --nnodes=1 --nproc-per-node=$(nvidia-smi -L | wc -l) te_layer_with_ov
 #   [rank0:node0] |-- Created tensor-parallel group: [0, 1, 2, 3, 4, 5, 6, 7]
 #   !!! [UB] Create UbufP2PCommOverlap Communicator
 #   UB_TIMEOUT is set to 110 sec, 217800000000 cycles, freq: 1980000khz
-#   MC initialized succesfully, window size = 549755813888
+#   MC initialized successfully, window size = 549755813888
 #   !!! [UBP2P] Register UBuf 1
 #   !!! [UBP2P] Register UBuf 2
 #   !!! [UBP2P] Register UBuf 3
@@ -66,7 +66,7 @@ $ torchrun --nnodes=1 --nproc-per-node=$(nvidia-smi -L | wc -l) te_layer_with_ov
 ```
 ### Single node, mixed data- and tensor-parallel LayerNormMLP:
 
-Uses `torch.nn.parallel.DistributedDataParallel` for replicatin the model across 2 tensor-parallel
+Uses `torch.nn.parallel.DistributedDataParallel` for replicating the model across 2 tensor-parallel
 groups in a single node.
 
 ```bash
@@ -81,7 +81,7 @@ $ torchrun --nnodes=1 --nproc-per-node=$(nvidia-smi -L | wc -l) te_layer_with_ov
 #   [rank2:node0] |-- Created data-parallel group: [2, 6]
 #   !!! [UB] Create UbufP2PCommOverlap Communicator
 #   UB_TIMEOUT is set to 110 sec, 217800000000 cycles, freq: 1980000khz
-#   MC initialized succesfully, window size = 549755813888
+#   MC initialized successfully, window size = 549755813888
 #   !!! [UBP2P] Register UBuf 1
 #   !!! [UBP2P] Register UBuf 2
 #   !!! [UBP2P] Register UBuf 3

examples/pytorch/comm_gemm_overlap/README.md

@@ -42,6 +42,11 @@
         shutil.rmtree(build_tools_copy)
     shutil.copytree(build_tools_dir, build_tools_copy)
 
+license_src = current_file_path.parent.parent / "LICENSE"
+license_dst = current_file_path / "LICENSE"
+if license_src.is_file():
+    shutil.copyfile(license_src, license_dst)
+
 
 from build_tools.build_ext import get_build_ext
 from build_tools.utils import copy_common_headers, min_python_version_str
@@ -131,7 +136,10 @@ def get_cuda_major_version() -> int:
         python_requires=f">={min_python_version_str()}",
         install_requires=install_requires,
         tests_require=test_requirements(),
+        license_files=("LICENSE",),
     )
     if any(x in sys.argv for x in (".", "sdist", "bdist_wheel")):
         shutil.rmtree(common_headers_dir)
         shutil.rmtree("build_tools")
+        if license_dst.is_file():
+            license_dst.unlink()

transformer_engine/jax/setup.py

@@ -43,6 +43,11 @@
         shutil.rmtree(build_tools_copy)
     shutil.copytree(build_tools_dir, build_tools_copy)
 
+license_src = current_file_path.parent.parent / "LICENSE"
+license_dst = current_file_path / "LICENSE"
+if license_src.is_file():
+    shutil.copyfile(license_src, license_dst)
+
 
 from build_tools.build_ext import get_build_ext
 from build_tools.utils import copy_common_headers, min_python_version_str
@@ -177,7 +182,10 @@ def run(self):
         python_requires=f">={min_python_version_str()}",
         install_requires=install_requires,
         tests_require=test_requirements(),
+        license_files=("LICENSE",),
     )
     if any(x in sys.argv for x in (".", "sdist", "bdist_wheel")):
         shutil.rmtree(common_headers_dir)
         shutil.rmtree("build_tools")
+        if license_dst.is_file():
+            license_dst.unlink()