diff --git a/CMakeLists.txt b/CMakeLists.txt
index 82794b185c3c..a11a8729700f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -344,9 +344,9 @@ tvm_file_glob(GLOB RUNTIME_SRCS
   src/runtime/*.cc
   src/runtime/vm/*.cc
   src/runtime/memory/*.cc
-  src/runtime/minrpc/*.cc
+  src/runtime/rpc/minrpc/*.cc
 )
-# Note: src/runtime/disco/** moves to libtvm_runtime_extra.
+# Note: src/runtime/extra/disco/** moves to libtvm_runtime_extra.
 # Note: src/runtime/{cuda,vulkan,opencl,metal,rocm,hexagon}/* move to per-backend DSOs.
 set(TVM_RUNTIME_EXT_OBJS "")
 
@@ -455,14 +455,14 @@ include(cmake/modules/Git.cmake)
 
 # ---- libtvm_runtime_extra assembly ----
 # Disco core sources.
-tvm_file_glob(GLOB _disco_core_srcs src/runtime/disco/*.cc)
+tvm_file_glob(GLOB _disco_core_srcs src/runtime/extra/disco/*.cc)
 add_library(tvm_disco_objs OBJECT ${_disco_core_srcs})
 target_link_libraries(tvm_disco_objs PRIVATE tvm_runtime_extra_defs)
 target_link_libraries(tvm_runtime_extra PRIVATE tvm_disco_objs)
 
 # Distributed disco (disabled for Hexagon cross-compile).
 if(NOT BUILD_FOR_HEXAGON)
-  tvm_file_glob(GLOB _disco_dist_srcs src/runtime/disco/distributed/*.cc)
+  tvm_file_glob(GLOB _disco_dist_srcs src/runtime/extra/disco/distributed/*.cc)
   add_library(tvm_disco_distributed_objs OBJECT ${_disco_dist_srcs})
   target_link_libraries(tvm_disco_distributed_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_disco_distributed_objs)
@@ -472,8 +472,8 @@ endif()
 if(USE_CUDA AND USE_NCCL)
   find_nccl(${USE_NCCL})
   include_directories(SYSTEM ${NCCL_INCLUDE_DIR})
-  tvm_file_glob(GLOB _nccl_srcs src/runtime/disco/nccl/*.cc src/runtime/disco/cuda_ipc/*.cc 3rdparty/tensorrt_llm/*.cu)
-  set_source_files_properties(src/runtime/disco/nccl/nccl.cc PROPERTIES COMPILE_DEFINITIONS "TVM_NCCL_RCCL_SWITCH=0")
+  tvm_file_glob(GLOB _nccl_srcs src/runtime/extra/disco/nccl/*.cc src/runtime/extra/disco/cuda_ipc/*.cc 3rdparty/tensorrt_llm/*.cu)
+  set_source_files_properties(src/runtime/extra/disco/nccl/nccl.cc PROPERTIES COMPILE_DEFINITIONS "TVM_NCCL_RCCL_SWITCH=0")
   add_library(tvm_nccl_objs OBJECT ${_nccl_srcs})
   target_link_libraries(tvm_nccl_objs PRIVATE tvm_runtime_extra_defs)
   find_library(LIBRT rt)
@@ -488,7 +488,7 @@ if(USE_CUDA AND USE_NVSHMEM)
   endif()
   set(CMAKE_CUDA_SEPARABLE_COMPILATION ON)
   set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-  tvm_file_glob(GLOB _nvshmem_srcs src/runtime/contrib/nvshmem/*.cc src/runtime/contrib/nvshmem/*.cu)
+  tvm_file_glob(GLOB _nvshmem_srcs src/runtime/extra/contrib/nvshmem/*.cc src/runtime/extra/contrib/nvshmem/*.cu)
   add_library(tvm_nvshmem_objs OBJECT ${_nvshmem_srcs})
   target_link_libraries(tvm_nvshmem_objs PRIVATE tvm_runtime_extra_defs)
   target_include_directories(tvm_nvshmem_objs PUBLIC ${NVSHMEM_INCLUDE_DIR})
@@ -502,8 +502,8 @@ endif()
 if(USE_ROCM AND USE_RCCL)
   find_rccl(${USE_RCCL})
   include_directories(SYSTEM ${RCCL_INCLUDE_DIR})
-  tvm_file_glob(GLOB _rccl_srcs src/runtime/disco/nccl/*.cc)
-  set_source_files_properties(src/runtime/disco/nccl/nccl.cc PROPERTIES COMPILE_DEFINITIONS "TVM_NCCL_RCCL_SWITCH=1")
+  tvm_file_glob(GLOB _rccl_srcs src/runtime/extra/disco/nccl/*.cc)
+  set_source_files_properties(src/runtime/extra/disco/nccl/nccl.cc PROPERTIES COMPILE_DEFINITIONS "TVM_NCCL_RCCL_SWITCH=1")
   add_library(tvm_rccl_objs OBJECT ${_rccl_srcs})
   target_link_libraries(tvm_rccl_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_rccl_objs rccl)
diff --git a/apps/android_rpc/app/src/main/jni/tvm_runtime.h b/apps/android_rpc/app/src/main/jni/tvm_runtime.h
index 920ae6bb1daf..ea9c7eb27895 100644
--- a/apps/android_rpc/app/src/main/jni/tvm_runtime.h
+++ b/apps/android_rpc/app/src/main/jni/tvm_runtime.h
@@ -40,7 +40,6 @@
 #include "../3rdparty/tvm-ffi/src/ffi/extra/library_module_dynamic_lib.cc"
 #include "../3rdparty/tvm-ffi/src/ffi/extra/library_module_system_lib.cc"
 #include "../3rdparty/tvm-ffi/src/ffi/extra/module.cc"
-#include "../3rdparty/tvm-ffi/src/ffi/extra/testing.cc"
 #include "../3rdparty/tvm-ffi/src/ffi/function.cc"
 #include "../3rdparty/tvm-ffi/src/ffi/object.cc"
 #include "../3rdparty/tvm-ffi/src/ffi/tensor.cc"
@@ -49,7 +48,6 @@
 #include "../src/runtime/file_utils.cc"
 #include "../src/runtime/logging.cc"
 #include "../src/runtime/memory/memory_manager.cc"
-#include "../src/runtime/minrpc/minrpc_logger.cc"
 #include "../src/runtime/registry.cc"
 #include "../src/runtime/rpc/rpc_channel.cc"
 #include "../src/runtime/rpc/rpc_endpoint.cc"
@@ -85,11 +83,11 @@
 #endif
 
 #ifdef USE_SORT
-#include "../src/runtime/contrib/sort/sort.cc"
+#include "../src/runtime/extra/contrib/sort/sort.cc"
 #endif
 
 #ifdef USE_RANDOM
-#include "../src/runtime/contrib/random/random.cc"
+#include "../src/runtime/extra/contrib/random/random.cc"
 #endif
 
 #include <android/log.h>
diff --git a/cmake/modules/CUDA.cmake b/cmake/modules/CUDA.cmake
index 4492cef90056..ec6160e7afaf 100644
--- a/cmake/modules/CUDA.cmake
+++ b/cmake/modules/CUDA.cmake
@@ -98,7 +98,7 @@ if(USE_CUDA AND USE_CUDNN)
   include_directories(SYSTEM ${CUDA_CUDNN_INCLUDE_DIRS})
   tvm_file_glob(GLOB CUDNN_RELAX_CONTRIB_SRC src/relax/backend/contrib/cudnn/*.cc)
   list(APPEND COMPILER_SRCS ${CUDNN_RELAX_CONTRIB_SRC})
-  tvm_file_glob(GLOB CONTRIB_CUDNN_SRCS src/runtime/contrib/cudnn/*.cc)
+  tvm_file_glob(GLOB CONTRIB_CUDNN_SRCS src/runtime/extra/contrib/cudnn/*.cc)
   add_library(tvm_cudnn_objs OBJECT ${CONTRIB_CUDNN_SRCS})
   target_link_libraries(tvm_cudnn_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_cudnn_objs ${CUDA_CUDNN_LIBRARY})
@@ -115,7 +115,7 @@ if(USE_CUDA AND USE_CUDNN_FRONTEND)
   if(NOT CUDNN_FRONTEND_HEADER)
     message(FATAL_ERROR "Cannot find cudnn_frontend.h, please set USE_CUDNN_FRONTEND to the path of the cuDNN frontend header")
   endif()
-  tvm_file_glob(GLOB CONTRIB_CUDNN_FRONTEND_SRCS src/runtime/contrib/cudnn/cudnn_frontend/*.cc)
+  tvm_file_glob(GLOB CONTRIB_CUDNN_FRONTEND_SRCS src/runtime/extra/contrib/cudnn/cudnn_frontend/*.cc)
   set_source_files_properties(${CONTRIB_CUDNN_SRCS} PROPERTIES COMPILE_DEFINITIONS TVM_USE_CUDNN_FRONTEND=1)
   add_library(tvm_cudnn_frontend_objs OBJECT ${CONTRIB_CUDNN_FRONTEND_SRCS})
   target_link_libraries(tvm_cudnn_frontend_objs PRIVATE tvm_runtime_extra_defs)
@@ -126,7 +126,7 @@ if(USE_CUDA AND USE_CUBLAS)
   message(STATUS "Build with cuBLAS support")
   tvm_file_glob(GLOB CUBLAS_CONTRIB_SRC src/relax/backend/contrib/cublas/*.cc)
   list(APPEND COMPILER_SRCS ${CUBLAS_CONTRIB_SRC})
-  tvm_file_glob(GLOB CONTRIB_CUBLAS_SRCS src/runtime/contrib/cublas/*.cc)
+  tvm_file_glob(GLOB CONTRIB_CUBLAS_SRCS src/runtime/extra/contrib/cublas/*.cc)
   add_library(tvm_cublas_objs OBJECT ${CONTRIB_CUBLAS_SRCS})
   target_link_libraries(tvm_cublas_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_cublas_objs ${CUDA_CUBLAS_LIBRARY})
@@ -137,7 +137,7 @@ endif(USE_CUDA AND USE_CUBLAS)
 
 if(USE_CUDA AND USE_THRUST)
   message(STATUS "Build with Thrust support")
-  tvm_file_glob(GLOB CONTRIB_THRUST_SRC src/runtime/contrib/thrust/*.cu)
+  tvm_file_glob(GLOB CONTRIB_THRUST_SRC src/runtime/extra/contrib/thrust/*.cu)
   add_library(tvm_thrust_objs OBJECT ${CONTRIB_THRUST_SRC})
   target_link_libraries(tvm_thrust_objs PRIVATE tvm_runtime_extra_defs)
   target_compile_options(tvm_thrust_objs PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>)
@@ -151,8 +151,8 @@ endif(USE_CUDA AND USE_THRUST)
 if(USE_CUDA AND USE_CURAND)
   message(STATUS "Build with cuRAND support")
   message(STATUS "${CUDA_CURAND_LIBRARY}")
-  tvm_file_glob(GLOB CONTRIB_CURAND_SRC_CC src/runtime/contrib/curand/*.cc)
-  tvm_file_glob(GLOB CONTRIB_CURAND_SRC_CU src/runtime/contrib/curand/*.cu)
+  tvm_file_glob(GLOB CONTRIB_CURAND_SRC_CC src/runtime/extra/contrib/curand/*.cc)
+  tvm_file_glob(GLOB CONTRIB_CURAND_SRC_CU src/runtime/extra/contrib/curand/*.cu)
   add_library(tvm_curand_objs OBJECT ${CONTRIB_CURAND_SRC_CC} ${CONTRIB_CURAND_SRC_CU})
   target_link_libraries(tvm_curand_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_curand_objs ${CUDA_CURAND_LIBRARY})
diff --git a/cmake/modules/Hexagon.cmake b/cmake/modules/Hexagon.cmake
index 9ddd677a6679..431b15b13ac6 100644
--- a/cmake/modules/Hexagon.cmake
+++ b/cmake/modules/Hexagon.cmake
@@ -289,8 +289,8 @@ if(USE_HEXAGON_RPC)
 
     # Include the generic RPC code into the TVM runtime.
     list(APPEND RUNTIME_HEXAGON_SRCS
-      "${TVMRT_SOURCE_DIR}/minrpc/minrpc_server.h"
-      "${TVMRT_SOURCE_DIR}/minrpc/rpc_reference.h"
+      "${TVMRT_SOURCE_DIR}/rpc/minrpc/minrpc_server.h"
+      "${TVMRT_SOURCE_DIR}/rpc/minrpc/rpc_reference.h"
       "${TVMRT_SOURCE_DIR}/rpc/rpc_module.cc"
       "${TVMRT_SOURCE_DIR}/rpc/rpc_endpoint.cc"
       "${TVMRT_SOURCE_DIR}/rpc/rpc_session.cc"
diff --git a/cmake/modules/ROCM.cmake b/cmake/modules/ROCM.cmake
index d502484cc7b0..b974aa412959 100644
--- a/cmake/modules/ROCM.cmake
+++ b/cmake/modules/ROCM.cmake
@@ -64,7 +64,7 @@ if(USE_ROCM AND USE_HIPBLAS)
   message(STATUS "Build with HIPBLAS support")
   tvm_file_glob(GLOB HIPBLAS_CONTRIB_SRC src/relax/backend/contrib/hipblas/*.cc)
   list(APPEND COMPILER_SRCS ${HIPBLAS_CONTRIB_SRC})
-  tvm_file_glob(GLOB HIPBLAS_CONTRIB_SRCS src/runtime/contrib/hipblas/*.cc)
+  tvm_file_glob(GLOB HIPBLAS_CONTRIB_SRCS src/runtime/extra/contrib/hipblas/*.cc)
   add_library(tvm_hipblas_objs OBJECT ${HIPBLAS_CONTRIB_SRCS})
   target_link_libraries(tvm_hipblas_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_hipblas_objs ${ROCM_HIPBLAS_LIBRARY})
@@ -84,8 +84,8 @@ if(USE_ROCM AND USE_THRUST)
 
   find_package(rocprim REQUIRED)
   find_package(rocthrust REQUIRED)
-  set_source_files_properties(src/runtime/contrib/thrust/thrust.cu PROPERTIES LANGUAGE CXX)
-  add_library(tvm_rocthrust_objs OBJECT src/runtime/contrib/thrust/thrust.cu)
+  set_source_files_properties(src/runtime/extra/contrib/thrust/thrust.cu PROPERTIES LANGUAGE CXX)
+  add_library(tvm_rocthrust_objs OBJECT src/runtime/extra/contrib/thrust/thrust.cu)
   target_link_libraries(tvm_rocthrust_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_rocthrust_objs roc::rocthrust)
 endif(USE_ROCM AND USE_THRUST)
diff --git a/cmake/modules/contrib/AMX.cmake b/cmake/modules/contrib/AMX.cmake
index ac349c4336a2..f01377417a4f 100644
--- a/cmake/modules/contrib/AMX.cmake
+++ b/cmake/modules/contrib/AMX.cmake
@@ -16,7 +16,7 @@
 # under the License.
 
 if(USE_AMX)
-    file(GLOB AMX_RUNTIME_CONFIG src/runtime/contrib/amx/amx_config.cc)
+    file(GLOB AMX_RUNTIME_CONFIG src/runtime/extra/contrib/amx/amx_config.cc)
     list(APPEND COMPILER_SRCS ${AMX_RUNTIME_CONFIG})
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=sapphirerapids")
     message(STATUS "Build with Intel AMX support...")
diff --git a/cmake/modules/contrib/BLAS.cmake b/cmake/modules/contrib/BLAS.cmake
index cee3e2fc30e7..23bb44446fee 100644
--- a/cmake/modules/contrib/BLAS.cmake
+++ b/cmake/modules/contrib/BLAS.cmake
@@ -17,7 +17,7 @@
 
 if(USE_BLAS STREQUAL "openblas")
   find_library(BLAS_LIBRARY openblas)
-  add_library(tvm_blas_objs OBJECT src/runtime/contrib/cblas/cblas.cc)
+  add_library(tvm_blas_objs OBJECT src/runtime/extra/contrib/cblas/cblas.cc)
   target_link_libraries(tvm_blas_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_blas_objs ${BLAS_LIBRARY})
   message(STATUS "Using BLAS library " ${BLAS_LIBRARY})
@@ -28,14 +28,14 @@ if(USE_BLAS STREQUAL "openblas")
   endif()
 elseif(USE_BLAS STREQUAL "atlas" OR USE_BLAS STREQUAL "blas")
   find_library(BLAS_LIBRARY cblas)
-  add_library(tvm_blas_objs OBJECT src/runtime/contrib/cblas/cblas.cc)
+  add_library(tvm_blas_objs OBJECT src/runtime/extra/contrib/cblas/cblas.cc)
   target_link_libraries(tvm_blas_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_blas_objs ${BLAS_LIBRARY})
   message(STATUS "Use BLAS library " ${BLAS_LIBRARY})
 elseif(USE_BLAS STREQUAL "apple")
   find_library(BLAS_LIBRARY Accelerate)
   include_directories(SYSTEM ${BLAS_LIBRARY}/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/)
-  add_library(tvm_blas_objs OBJECT src/runtime/contrib/cblas/cblas.cc)
+  add_library(tvm_blas_objs OBJECT src/runtime/extra/contrib/cblas/cblas.cc)
   target_link_libraries(tvm_blas_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_blas_objs ${BLAS_LIBRARY})
   message(STATUS "Use BLAS library " ${BLAS_LIBRARY})
@@ -66,7 +66,7 @@ if(USE_MKL OR USE_MKL_PATH)
     find_library(BLAS_LIBRARY_MKL NAMES mkl_rt HINTS ${USE_MKL}/lib/ ${USE_MKL}/lib/intel64_win)
   endif()
   include_directories(SYSTEM ${USE_MKL}/include)
-  add_library(tvm_mkl_objs OBJECT src/runtime/contrib/cblas/mkl.cc)
+  add_library(tvm_mkl_objs OBJECT src/runtime/extra/contrib/cblas/mkl.cc)
   target_link_libraries(tvm_mkl_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_mkl_objs ${BLAS_LIBRARY_MKL})
   add_definitions(-DUSE_MKL_BLAS=1)
diff --git a/cmake/modules/contrib/CLML.cmake b/cmake/modules/contrib/CLML.cmake
index 1d0f0f3a50cf..2f12713b139e 100644
--- a/cmake/modules/contrib/CLML.cmake
+++ b/cmake/modules/contrib/CLML.cmake
@@ -17,7 +17,7 @@
 
 if(USE_CLML)
     file(GLOB CLML_RELAX_CONTRIB_SRC src/relax/backend/contrib/clml/*.cc)
-    file(GLOB CLML_RUNTIME_MODULE src/runtime/contrib/clml/clml_runtime.cc)
+    file(GLOB CLML_RUNTIME_MODULE src/runtime/extra/contrib/clml/clml_runtime.cc)
     include_directories(SYSTEM "3rdparty/OpenCL-Headers")
     list(APPEND COMPILER_SRCS ${CLML_RELAX_CONTRIB_SRC})
     if(NOT USE_CLML_GRAPH_EXECUTOR)
@@ -49,7 +49,7 @@ if(USE_CLML_GRAPH_EXECUTOR)
         set(CLML_PATH ${USE_CLML_GRAPH_EXECUTOR})
     endif()
 
-    file(GLOB CLML_CONTRIB_SRC src/runtime/contrib/clml/*)
+    file(GLOB CLML_CONTRIB_SRC src/runtime/extra/contrib/clml/*)
 
     # CMake needs to find clml library, include and support directories
     # in the path specified by CLML_PATH.
diff --git a/cmake/modules/contrib/CUTLASS.cmake b/cmake/modules/contrib/CUTLASS.cmake
index 7f44c2e6db0c..6d81ba923813 100644
--- a/cmake/modules/contrib/CUTLASS.cmake
+++ b/cmake/modules/contrib/CUTLASS.cmake
@@ -33,7 +33,7 @@ if(USE_CUDA AND USE_CUTLASS)
   target_link_libraries(fpA_intB_gemm_tvm PRIVATE tvm_ffi_header)
 
   set(CUTLASS_FPA_INTB_RUNTIME_SRCS "")
-  list(APPEND CUTLASS_FPA_INTB_RUNTIME_SRCS src/runtime/contrib/cutlass/weight_preprocess.cc)
+  list(APPEND CUTLASS_FPA_INTB_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/weight_preprocess.cc)
   add_library(fpA_intB_cutlass_objs OBJECT ${CUTLASS_FPA_INTB_RUNTIME_SRCS})
   target_link_libraries(fpA_intB_cutlass_objs PRIVATE tvm_runtime_extra_defs)
   target_include_directories(fpA_intB_cutlass_objs PRIVATE
@@ -54,15 +54,15 @@ if(USE_CUDA AND USE_CUTLASS)
   set(TVM_CUTLASS_RUNTIME_SRCS "")
 
   if(CMAKE_CUDA_ARCHITECTURES MATCHES "90a")
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp16_group_gemm_sm90.cu)
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp8_group_gemm_sm90.cu)
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp8_gemm.cu)
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm90.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp8_group_gemm_sm90.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp8_gemm.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu)
   endif()
   if(CMAKE_CUDA_ARCHITECTURES MATCHES "100a")
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp16_group_gemm_sm100.cu)
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu)
-    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm100.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu)
+    list(APPEND TVM_CUTLASS_RUNTIME_SRCS src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu)
   endif()
   if(TVM_CUTLASS_RUNTIME_SRCS)
     add_library(tvm_cutlass_objs OBJECT ${TVM_CUTLASS_RUNTIME_SRCS})
diff --git a/cmake/modules/contrib/CoreML.cmake b/cmake/modules/contrib/CoreML.cmake
index 94520f2b570f..fdadf4f3d602 100644
--- a/cmake/modules/contrib/CoreML.cmake
+++ b/cmake/modules/contrib/CoreML.cmake
@@ -19,7 +19,7 @@ if(USE_COREML)
   message(STATUS "Build with contrib.coreml")
   find_library(FOUNDATION_LIB Foundation)
   find_library(COREML_LIB Coreml)
-  tvm_file_glob(GLOB COREML_CONTRIB_SRC src/runtime/contrib/coreml/*.mm)
+  tvm_file_glob(GLOB COREML_CONTRIB_SRC src/runtime/extra/contrib/coreml/*.mm)
   add_library(tvm_coreml_objs OBJECT ${COREML_CONTRIB_SRC})
   target_link_libraries(tvm_coreml_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_coreml_objs ${FOUNDATION_LIB} ${COREML_LIB})
diff --git a/cmake/modules/contrib/DNNL.cmake b/cmake/modules/contrib/DNNL.cmake
index 191b04594b80..087e0e0cc994 100644
--- a/cmake/modules/contrib/DNNL.cmake
+++ b/cmake/modules/contrib/DNNL.cmake
@@ -24,10 +24,10 @@ if(IS_DIRECTORY ${USE_DNNL})
     tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/relax/backend/contrib/dnnl/*.cc)
     list(APPEND COMPILER_SRCS ${DNNL_CONTRIB_SRC})
 
-    tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/contrib/dnnl/dnnl_json_runtime.cc
-                                        src/runtime/contrib/dnnl/dnnl_utils.cc
-                                        src/runtime/contrib/dnnl/dnnl.cc
-                                        src/runtime/contrib/cblas/dnnl_blas.cc)
+    tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/extra/contrib/dnnl/dnnl_json_runtime.cc
+                                        src/runtime/extra/contrib/dnnl/dnnl_utils.cc
+                                        src/runtime/extra/contrib/dnnl/dnnl.cc
+                                        src/runtime/extra/contrib/cblas/dnnl_blas.cc)
     add_library(tvm_dnnl_objs OBJECT ${DNNL_CONTRIB_SRC})
     target_link_libraries(tvm_dnnl_objs PRIVATE tvm_runtime_extra_defs)
     target_link_libraries(tvm_runtime_extra PRIVATE tvm_dnnl_objs ${EXTERN_LIBRARY_DNNL})
@@ -39,19 +39,19 @@ elseif((USE_DNNL STREQUAL "ON") OR (USE_DNNL STREQUAL "JSON"))
   list(APPEND COMPILER_SRCS ${DNNL_CONTRIB_SRC})
 
   find_library(EXTERN_LIBRARY_DNNL dnnl)
-  tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/contrib/dnnl/dnnl_json_runtime.cc
-                                      src/runtime/contrib/dnnl/dnnl_utils.cc
-                                      src/runtime/contrib/dnnl/dnnl.cc
-                                      src/runtime/contrib/cblas/dnnl_blas.cc)
+  tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/extra/contrib/dnnl/dnnl_json_runtime.cc
+                                      src/runtime/extra/contrib/dnnl/dnnl_utils.cc
+                                      src/runtime/extra/contrib/dnnl/dnnl.cc
+                                      src/runtime/extra/contrib/cblas/dnnl_blas.cc)
   add_library(tvm_dnnl_objs OBJECT ${DNNL_CONTRIB_SRC})
   target_link_libraries(tvm_dnnl_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_dnnl_objs ${EXTERN_LIBRARY_DNNL})
   message(STATUS "Build with DNNL JSON runtime: " ${EXTERN_LIBRARY_DNNL})
 elseif(USE_DNNL STREQUAL "C_SRC")
   find_library(EXTERN_LIBRARY_DNNL dnnl)
-  tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/contrib/dnnl/dnnl.cc
-                                      src/runtime/contrib/dnnl/dnnl_utils.cc
-                                      src/runtime/contrib/cblas/dnnl_blas.cc)
+  tvm_file_glob(GLOB DNNL_CONTRIB_SRC src/runtime/extra/contrib/dnnl/dnnl.cc
+                                      src/runtime/extra/contrib/dnnl/dnnl_utils.cc
+                                      src/runtime/extra/contrib/cblas/dnnl_blas.cc)
   add_library(tvm_dnnl_objs OBJECT ${DNNL_CONTRIB_SRC})
   target_link_libraries(tvm_dnnl_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_dnnl_objs ${EXTERN_LIBRARY_DNNL})
diff --git a/cmake/modules/contrib/ExampleNPU.cmake b/cmake/modules/contrib/ExampleNPU.cmake
index 51b023dbbf0d..b2567a44fd41 100644
--- a/cmake/modules/contrib/ExampleNPU.cmake
+++ b/cmake/modules/contrib/ExampleNPU.cmake
@@ -22,7 +22,7 @@ if(USE_EXAMPLE_NPU_CODEGEN)
     tvm_file_glob(GLOB COMPILER_EXAMPLE_NPU_SRCS src/relax/backend/contrib/example_npu/*.cc)
     list(APPEND COMPILER_SRCS ${COMPILER_EXAMPLE_NPU_SRCS})
 
-    tvm_file_glob(GLOB RUNTIME_EXAMPLE_NPU_SRCS src/runtime/contrib/example_npu/*.cc)
+    tvm_file_glob(GLOB RUNTIME_EXAMPLE_NPU_SRCS src/runtime/extra/contrib/example_npu/*.cc)
     if(NOT USE_EXAMPLE_NPU_RUNTIME)
         list(APPEND COMPILER_SRCS ${RUNTIME_EXAMPLE_NPU_SRCS})
     endif()
@@ -32,7 +32,7 @@ endif()
 if(USE_EXAMPLE_NPU_RUNTIME)
     message(STATUS "Build with Example NPU runtime")
 
-    tvm_file_glob(GLOB RUNTIME_EXAMPLE_NPU_SRCS src/runtime/contrib/example_npu/*.cc)
+    tvm_file_glob(GLOB RUNTIME_EXAMPLE_NPU_SRCS src/runtime/extra/contrib/example_npu/*.cc)
     add_library(tvm_example_npu_objs OBJECT ${RUNTIME_EXAMPLE_NPU_SRCS})
     target_link_libraries(tvm_example_npu_objs PRIVATE tvm_runtime_extra_defs)
     target_link_libraries(tvm_runtime_extra PRIVATE tvm_example_npu_objs)
diff --git a/cmake/modules/contrib/NNAPI.cmake b/cmake/modules/contrib/NNAPI.cmake
index 496ce96a8060..6d1c44b8425a 100644
--- a/cmake/modules/contrib/NNAPI.cmake
+++ b/cmake/modules/contrib/NNAPI.cmake
@@ -20,7 +20,7 @@ if(USE_NNAPI_CODEGEN)
     message(STATUS "Build with NNAPI codegen")
 
     tvm_file_glob(GLOB COMPILER_NNAPI_SRCS src/relax/backend/contrib/nnapi/*.cc)
-    tvm_file_glob(GLOB RUNTIME_NNAPI_SRCS src/runtime/contrib/nnapi/*.cc)
+    tvm_file_glob(GLOB RUNTIME_NNAPI_SRCS src/runtime/extra/contrib/nnapi/*.cc)
     list(APPEND COMPILER_SRCS ${COMPILER_NNAPI_SRCS})
     if(NOT USE_NNAPI_RUNTIME)
         list(APPEND COMPILER_SRCS ${RUNTIME_NNAPI_SRCS})
@@ -31,7 +31,7 @@ endif()
 if(USE_NNAPI_RUNTIME)
     message(STATUS "Build with NNAPI runtime")
 
-    tvm_file_glob(GLOB RUNTIME_NNAPI_SRCS src/runtime/contrib/nnapi/*.cc)
+    tvm_file_glob(GLOB RUNTIME_NNAPI_SRCS src/runtime/extra/contrib/nnapi/*.cc)
     add_library(tvm_nnapi_objs OBJECT ${RUNTIME_NNAPI_SRCS})
     target_link_libraries(tvm_nnapi_objs PRIVATE tvm_runtime_extra_defs)
     target_link_libraries(tvm_runtime_extra PRIVATE tvm_nnapi_objs neuralnetworks log)
diff --git a/cmake/modules/contrib/Random.cmake b/cmake/modules/contrib/Random.cmake
index 16e699fb62a6..215b73e0fb84 100644
--- a/cmake/modules/contrib/Random.cmake
+++ b/cmake/modules/contrib/Random.cmake
@@ -17,7 +17,7 @@
 
 if(USE_RANDOM)
   message(STATUS "Build with contrib.random")
-  tvm_file_glob(GLOB RANDOM_CONTRIB_SRC src/runtime/contrib/random/random.cc)
+  tvm_file_glob(GLOB RANDOM_CONTRIB_SRC src/runtime/extra/contrib/random/random.cc)
   add_library(tvm_random_objs OBJECT ${RANDOM_CONTRIB_SRC})
   target_link_libraries(tvm_random_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_random_objs)
diff --git a/cmake/modules/contrib/Sort.cmake b/cmake/modules/contrib/Sort.cmake
index 2fbeedd95e30..0c02fe861e1d 100644
--- a/cmake/modules/contrib/Sort.cmake
+++ b/cmake/modules/contrib/Sort.cmake
@@ -17,7 +17,7 @@
 
 if(USE_SORT)
   message(STATUS "Build with contrib.sort")
-  tvm_file_glob(GLOB SORT_CONTRIB_SRC src/runtime/contrib/sort/*.cc)
+  tvm_file_glob(GLOB SORT_CONTRIB_SRC src/runtime/extra/contrib/sort/*.cc)
   add_library(tvm_sort_objs OBJECT ${SORT_CONTRIB_SRC})
   target_link_libraries(tvm_sort_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_sort_objs)
diff --git a/cmake/modules/contrib/TensorRT.cmake b/cmake/modules/contrib/TensorRT.cmake
index 08a841a46df1..0b70d7757282 100644
--- a/cmake/modules/contrib/TensorRT.cmake
+++ b/cmake/modules/contrib/TensorRT.cmake
@@ -25,7 +25,7 @@ if(USE_TENSORRT_CODEGEN)
     message(STATUS "Build with TensorRT codegen")
     tvm_file_glob(GLOB COMPILER_TENSORRT_SRCS src/relax/backend/contrib/tensorrt/*.cc)
     set_source_files_properties(${COMPILER_TENSORRT_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-deprecated-declarations")
-    tvm_file_glob(GLOB RUNTIME_TENSORRT_SRCS src/runtime/contrib/tensorrt/tensorrt_runtime.cc)
+    tvm_file_glob(GLOB RUNTIME_TENSORRT_SRCS src/runtime/extra/contrib/tensorrt/tensorrt_runtime.cc)
     set_source_files_properties(${RUNTIME_TENSORRT_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-deprecated-declarations")
     list(APPEND COMPILER_SRCS ${COMPILER_TENSORRT_SRCS})
     if(NOT USE_TENSORRT_RUNTIME)
@@ -48,7 +48,7 @@ if(USE_TENSORRT_RUNTIME)
     message(STATUS "TENSORRT_LIB_DIR: " ${TENSORRT_LIB_DIR})
     include_directories(${TENSORRT_INCLUDE_DIR})
     # TRT runtime sources
-    tvm_file_glob(GLOB RUNTIME_TENSORRT_SRCS src/runtime/contrib/tensorrt/*.cc)
+    tvm_file_glob(GLOB RUNTIME_TENSORRT_SRCS src/runtime/extra/contrib/tensorrt/*.cc)
     set_source_files_properties(${RUNTIME_TENSORRT_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-deprecated-declarations")
     add_library(tvm_tensorrt_objs OBJECT ${RUNTIME_TENSORRT_SRCS})
     target_link_libraries(tvm_tensorrt_objs PRIVATE tvm_runtime_extra_defs)
diff --git a/cmake/modules/contrib/vllm.cmake b/cmake/modules/contrib/vllm.cmake
index 7a571ff50508..56b452fe7db4 100644
--- a/cmake/modules/contrib/vllm.cmake
+++ b/cmake/modules/contrib/vllm.cmake
@@ -17,10 +17,10 @@
 
 if(USE_VLLM)
   message(STATUS "Build with vllm paged attention kernel.")
-  include_directories(src/runtime/contrib/vllm)
+  include_directories(src/runtime/extra/contrib/vllm)
   enable_language(CUDA)
 
-  tvm_file_glob(GLOB VLLM_CONTRIB_SRC src/runtime/contrib/vllm/*.cu src/runtime/contrib/vllm/*.cc)
+  tvm_file_glob(GLOB VLLM_CONTRIB_SRC src/runtime/extra/contrib/vllm/*.cu src/runtime/extra/contrib/vllm/*.cc)
   add_library(tvm_vllm_objs OBJECT ${VLLM_CONTRIB_SRC})
   target_link_libraries(tvm_vllm_objs PRIVATE tvm_runtime_extra_defs)
   target_link_libraries(tvm_runtime_extra PRIVATE tvm_vllm_objs)
diff --git a/python/tvm/rpc/minrpc.py b/python/tvm/rpc/minrpc.py
index 73a804d59204..f3819c27913b 100644
--- a/python/tvm/rpc/minrpc.py
+++ b/python/tvm/rpc/minrpc.py
@@ -39,7 +39,7 @@ def find_minrpc_server_libpath(server="posix_popen_server"):
     """
     curr_dir = os.path.dirname(os.path.realpath(os.path.expanduser(__file__)))
     source_dir = os.path.abspath(os.path.join(curr_dir, "..", "..", ".."))
-    minrpc_dir = os.path.join(source_dir, "src", "runtime", "minrpc")
+    minrpc_dir = os.path.join(source_dir, "src", "runtime", "rpc", "minrpc")
     path = os.path.join(minrpc_dir, server, f"{server}.cc")
 
     candidates = [path]
diff --git a/src/relax/backend/contrib/codegen_json/codegen_json.h b/src/relax/backend/contrib/codegen_json/codegen_json.h
index d7874eb84679..34ebdd8e9ec0 100644
--- a/src/relax/backend/contrib/codegen_json/codegen_json.h
+++ b/src/relax/backend/contrib/codegen_json/codegen_json.h
@@ -35,8 +35,8 @@
 #include <string>
 #include <vector>
 
-#include "../../../../runtime/contrib/json/json_node.h"
-#include "../../../../runtime/contrib/json/json_runtime.h"
+#include "../../../../runtime/extra/contrib/json/json_node.h"
+#include "../../../../runtime/extra/contrib/json/json_runtime.h"
 #include "../../../transform/utils.h"
 #include "../utils.h"
 
diff --git a/src/relax/transform/static_plan_block_memory.cc b/src/relax/transform/static_plan_block_memory.cc
index 73ddc0b46b9f..b8b6ba30d25b 100644
--- a/src/relax/transform/static_plan_block_memory.cc
+++ b/src/relax/transform/static_plan_block_memory.cc
@@ -78,7 +78,7 @@
 #include <set>
 #include <vector>
 
-#include "../../runtime/texture.h"
+#include "../../runtime/opencl/texture.h"
 #include "utils.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/amx/amx_config.cc b/src/runtime/extra/contrib/amx/amx_config.cc
similarity index 100%
rename from src/runtime/contrib/amx/amx_config.cc
rename to src/runtime/extra/contrib/amx/amx_config.cc
diff --git a/src/runtime/contrib/cblas/cblas.cc b/src/runtime/extra/contrib/cblas/cblas.cc
similarity index 100%
rename from src/runtime/contrib/cblas/cblas.cc
rename to src/runtime/extra/contrib/cblas/cblas.cc
diff --git a/src/runtime/contrib/cblas/dnnl_blas.cc b/src/runtime/extra/contrib/cblas/dnnl_blas.cc
similarity index 100%
rename from src/runtime/contrib/cblas/dnnl_blas.cc
rename to src/runtime/extra/contrib/cblas/dnnl_blas.cc
diff --git a/src/runtime/contrib/cblas/gemm_common.h b/src/runtime/extra/contrib/cblas/gemm_common.h
similarity index 100%
rename from src/runtime/contrib/cblas/gemm_common.h
rename to src/runtime/extra/contrib/cblas/gemm_common.h
diff --git a/src/runtime/contrib/cblas/mkl.cc b/src/runtime/extra/contrib/cblas/mkl.cc
similarity index 100%
rename from src/runtime/contrib/cblas/mkl.cc
rename to src/runtime/extra/contrib/cblas/mkl.cc
diff --git a/src/runtime/contrib/clml/clml_memory_planner.cc b/src/runtime/extra/contrib/clml/clml_memory_planner.cc
similarity index 100%
rename from src/runtime/contrib/clml/clml_memory_planner.cc
rename to src/runtime/extra/contrib/clml/clml_memory_planner.cc
diff --git a/src/runtime/contrib/clml/clml_memory_planner.h b/src/runtime/extra/contrib/clml/clml_memory_planner.h
similarity index 100%
rename from src/runtime/contrib/clml/clml_memory_planner.h
rename to src/runtime/extra/contrib/clml/clml_memory_planner.h
diff --git a/src/runtime/contrib/clml/clml_runtime.cc b/src/runtime/extra/contrib/clml/clml_runtime.cc
similarity index 99%
rename from src/runtime/contrib/clml/clml_runtime.cc
rename to src/runtime/extra/contrib/clml/clml_runtime.cc
index dd66987bdd11..e426545be2fd 100644
--- a/src/runtime/contrib/clml/clml_runtime.cc
+++ b/src/runtime/extra/contrib/clml/clml_runtime.cc
@@ -29,7 +29,7 @@
 
 #include <unordered_map>
 
-#include "../../../support/bytes_io.h"
+#include "../../../../support/bytes_io.h"
 
 #ifdef TVM_GRAPH_EXECUTOR_CLML
 #include "clml_memory_planner.h"
diff --git a/src/runtime/contrib/clml/clml_runtime.h b/src/runtime/extra/contrib/clml/clml_runtime.h
similarity index 99%
rename from src/runtime/contrib/clml/clml_runtime.h
rename to src/runtime/extra/contrib/clml/clml_runtime.h
index 3a0a7b12c01e..5de3fedaaf7a 100644
--- a/src/runtime/contrib/clml/clml_runtime.h
+++ b/src/runtime/extra/contrib/clml/clml_runtime.h
@@ -42,9 +42,9 @@
 #include <utility>
 #include <vector>
 
-#include "../../file_utils.h"
-#include "../../opencl/opencl_common.h"
-#include "../../thread_storage_scope.h"
+#include "../../../file_utils.h"
+#include "../../../opencl/opencl_common.h"
+#include "../../../thread_storage_scope.h"
 #include "../json/json_node.h"
 #include "../json/json_runtime.h"
 
diff --git a/src/runtime/contrib/clml/clml_utils.cc b/src/runtime/extra/contrib/clml/clml_utils.cc
similarity index 100%
rename from src/runtime/contrib/clml/clml_utils.cc
rename to src/runtime/extra/contrib/clml/clml_utils.cc
diff --git a/src/runtime/contrib/clml/clml_utils.h b/src/runtime/extra/contrib/clml/clml_utils.h
similarity index 100%
rename from src/runtime/contrib/clml/clml_utils.h
rename to src/runtime/extra/contrib/clml/clml_utils.h
diff --git a/src/runtime/contrib/coreml/coreml_runtime.h b/src/runtime/extra/contrib/coreml/coreml_runtime.h
similarity index 100%
rename from src/runtime/contrib/coreml/coreml_runtime.h
rename to src/runtime/extra/contrib/coreml/coreml_runtime.h
diff --git a/src/runtime/contrib/coreml/coreml_runtime.mm b/src/runtime/extra/contrib/coreml/coreml_runtime.mm
similarity index 99%
rename from src/runtime/contrib/coreml/coreml_runtime.mm
rename to src/runtime/extra/contrib/coreml/coreml_runtime.mm
index 5c0234e77228..82f7c51c9ea7 100644
--- a/src/runtime/contrib/coreml/coreml_runtime.mm
+++ b/src/runtime/extra/contrib/coreml/coreml_runtime.mm
@@ -23,7 +23,7 @@
 #include <tvm/ffi/function.h>
 #include <tvm/ffi/reflection/registry.h>
 
-#include "../../../support/bytes_io.h"
+#include "../../../../support/bytes_io.h"
 #include "coreml_runtime.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/cublas/cublas.cc b/src/runtime/extra/contrib/cublas/cublas.cc
similarity index 99%
rename from src/runtime/contrib/cublas/cublas.cc
rename to src/runtime/extra/contrib/cublas/cublas.cc
index e58ffdeee0ba..66e9a74c8675 100644
--- a/src/runtime/contrib/cublas/cublas.cc
+++ b/src/runtime/extra/contrib/cublas/cublas.cc
@@ -26,7 +26,7 @@
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/runtime/data_type.h>
 
-#include "../../3rdparty/compiler-rt/builtin_fp16.h"
+#include "../../../../../3rdparty/compiler-rt/builtin_fp16.h"
 #include "../cblas/gemm_common.h"
 #include "cublas_utils.h"
 
diff --git a/src/runtime/contrib/cublas/cublas_json_runtime.cc b/src/runtime/extra/contrib/cublas/cublas_json_runtime.cc
similarity index 99%
rename from src/runtime/contrib/cublas/cublas_json_runtime.cc
rename to src/runtime/extra/contrib/cublas/cublas_json_runtime.cc
index 34bcbdcf4976..f63d8575e05b 100644
--- a/src/runtime/contrib/cublas/cublas_json_runtime.cc
+++ b/src/runtime/extra/contrib/cublas/cublas_json_runtime.cc
@@ -32,7 +32,7 @@
 #include <string>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 #include "../json/json_node.h"
 #include "../json/json_runtime.h"
 #include "cublas_utils.h"
diff --git a/src/runtime/contrib/cublas/cublas_utils.cc b/src/runtime/extra/contrib/cublas/cublas_utils.cc
similarity index 98%
rename from src/runtime/contrib/cublas/cublas_utils.cc
rename to src/runtime/extra/contrib/cublas/cublas_utils.cc
index b18f7a14ef1b..5050f20998fa 100644
--- a/src/runtime/contrib/cublas/cublas_utils.cc
+++ b/src/runtime/extra/contrib/cublas/cublas_utils.cc
@@ -25,7 +25,7 @@
 #include <tvm/ffi/extra/c_env_api.h>
 #include <tvm/ffi/function.h>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 namespace tvm {
 namespace contrib {
diff --git a/src/runtime/contrib/cublas/cublas_utils.h b/src/runtime/extra/contrib/cublas/cublas_utils.h
similarity index 100%
rename from src/runtime/contrib/cublas/cublas_utils.h
rename to src/runtime/extra/contrib/cublas/cublas_utils.h
diff --git a/src/runtime/contrib/cudnn/conv_backward.cc b/src/runtime/extra/contrib/cudnn/conv_backward.cc
similarity index 100%
rename from src/runtime/contrib/cudnn/conv_backward.cc
rename to src/runtime/extra/contrib/cudnn/conv_backward.cc
diff --git a/src/runtime/contrib/cudnn/conv_forward.cc b/src/runtime/extra/contrib/cudnn/conv_forward.cc
similarity index 100%
rename from src/runtime/contrib/cudnn/conv_forward.cc
rename to src/runtime/extra/contrib/cudnn/conv_forward.cc
diff --git a/src/runtime/contrib/cudnn/cudnn_frontend/attention.cc b/src/runtime/extra/contrib/cudnn/cudnn_frontend/attention.cc
similarity index 99%
rename from src/runtime/contrib/cudnn/cudnn_frontend/attention.cc
rename to src/runtime/extra/contrib/cudnn/cudnn_frontend/attention.cc
index 5dea47176c1a..32f33fc739c1 100644
--- a/src/runtime/contrib/cudnn/cudnn_frontend/attention.cc
+++ b/src/runtime/extra/contrib/cudnn/cudnn_frontend/attention.cc
@@ -27,7 +27,7 @@
 #include <tvm/ffi/function.h>
 #include <tvm/runtime/device_api.h>
 
-#include "../../../cuda/cuda_common.h"
+#include "../../../../cuda/cuda_common.h"
 #include "../cudnn_utils.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/cudnn/cudnn_frontend/attention.h b/src/runtime/extra/contrib/cudnn/cudnn_frontend/attention.h
similarity index 100%
rename from src/runtime/contrib/cudnn/cudnn_frontend/attention.h
rename to src/runtime/extra/contrib/cudnn/cudnn_frontend/attention.h
diff --git a/src/runtime/contrib/cudnn/cudnn_json_runtime.cc b/src/runtime/extra/contrib/cudnn/cudnn_json_runtime.cc
similarity index 100%
rename from src/runtime/contrib/cudnn/cudnn_json_runtime.cc
rename to src/runtime/extra/contrib/cudnn/cudnn_json_runtime.cc
diff --git a/src/runtime/contrib/cudnn/cudnn_utils.cc b/src/runtime/extra/contrib/cudnn/cudnn_utils.cc
similarity index 100%
rename from src/runtime/contrib/cudnn/cudnn_utils.cc
rename to src/runtime/extra/contrib/cudnn/cudnn_utils.cc
diff --git a/src/runtime/contrib/cudnn/cudnn_utils.h b/src/runtime/extra/contrib/cudnn/cudnn_utils.h
similarity index 99%
rename from src/runtime/contrib/cudnn/cudnn_utils.h
rename to src/runtime/extra/contrib/cudnn/cudnn_utils.h
index 91f50dfc1c92..65ee263fdc4c 100644
--- a/src/runtime/contrib/cudnn/cudnn_utils.h
+++ b/src/runtime/extra/contrib/cudnn/cudnn_utils.h
@@ -30,7 +30,7 @@
 
 #include <string>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 namespace tvm {
 namespace contrib {
diff --git a/src/runtime/contrib/cudnn/softmax.cc b/src/runtime/extra/contrib/cudnn/softmax.cc
similarity index 100%
rename from src/runtime/contrib/cudnn/softmax.cc
rename to src/runtime/extra/contrib/cudnn/softmax.cc
diff --git a/src/runtime/contrib/curand/curand.cc b/src/runtime/extra/contrib/curand/curand.cc
similarity index 99%
rename from src/runtime/contrib/curand/curand.cc
rename to src/runtime/extra/contrib/curand/curand.cc
index 4dd0ca145c37..5dd4f2b4aa91 100644
--- a/src/runtime/contrib/curand/curand.cc
+++ b/src/runtime/extra/contrib/curand/curand.cc
@@ -21,7 +21,7 @@
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/runtime/base.h>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 #include "./helper_cuda_kernels.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/curand/helper_cuda_kernels.cu b/src/runtime/extra/contrib/curand/helper_cuda_kernels.cu
similarity index 100%
rename from src/runtime/contrib/curand/helper_cuda_kernels.cu
rename to src/runtime/extra/contrib/curand/helper_cuda_kernels.cu
diff --git a/src/runtime/contrib/curand/helper_cuda_kernels.h b/src/runtime/extra/contrib/curand/helper_cuda_kernels.h
similarity index 100%
rename from src/runtime/contrib/curand/helper_cuda_kernels.h
rename to src/runtime/extra/contrib/curand/helper_cuda_kernels.h
diff --git a/src/runtime/contrib/cutlass/fp16_group_gemm.cuh b/src/runtime/extra/contrib/cutlass/fp16_group_gemm.cuh
similarity index 100%
rename from src/runtime/contrib/cutlass/fp16_group_gemm.cuh
rename to src/runtime/extra/contrib/cutlass/fp16_group_gemm.cuh
diff --git a/src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh
rename to src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh
index 055eb543dc1d..b73ab99d07ad 100644
--- a/src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh
+++ b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm100.cuh
@@ -25,7 +25,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh
rename to src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh
index 16455efc00bd..5ab825a63995 100644
--- a/src/runtime/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh
+++ b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_runner_sm90.cuh
@@ -25,7 +25,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/fp16_group_gemm_sm100.cu b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm100.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp16_group_gemm_sm100.cu
rename to src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm100.cu
diff --git a/src/runtime/contrib/cutlass/fp16_group_gemm_sm90.cu b/src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm90.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp16_group_gemm_sm90.cu
rename to src/runtime/extra/contrib/cutlass/fp16_group_gemm_sm90.cu
diff --git a/src/runtime/contrib/cutlass/fp8_gemm.cu b/src/runtime/extra/contrib/cutlass/fp8_gemm.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_gemm.cu
rename to src/runtime/extra/contrib/cutlass/fp8_gemm.cu
diff --git a/src/runtime/contrib/cutlass/fp8_group_gemm_sm90.cu b/src/runtime/extra/contrib/cutlass/fp8_group_gemm_sm90.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_group_gemm_sm90.cu
rename to src/runtime/extra/contrib/cutlass/fp8_group_gemm_sm90.cu
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm.cuh b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm.cuh
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm.cuh
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm.cuh
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh
index 6348330f01c1..3f7f89ca6df7 100644
--- a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh
+++ b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm100.cuh
@@ -24,7 +24,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh
index 82be78330ea1..ee47a5f69283 100644
--- a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh
+++ b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_runner_sm90.cuh
@@ -24,7 +24,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm100.cu
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_gemm_sm90.cu
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh
index 47dbf65f9d29..0f7ffee5defc 100644
--- a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh
+++ b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_runner_sm100.cuh
@@ -23,7 +23,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu b/src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu
similarity index 100%
rename from src/runtime/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu
rename to src/runtime/extra/contrib/cutlass/fp8_groupwise_scaled_group_gemm_sm100.cu
diff --git a/src/runtime/contrib/cutlass/gemm_runner.cuh b/src/runtime/extra/contrib/cutlass/gemm_runner.cuh
similarity index 99%
rename from src/runtime/contrib/cutlass/gemm_runner.cuh
rename to src/runtime/extra/contrib/cutlass/gemm_runner.cuh
index 1e8fd40fb93b..5d876291f00e 100644
--- a/src/runtime/contrib/cutlass/gemm_runner.cuh
+++ b/src/runtime/extra/contrib/cutlass/gemm_runner.cuh
@@ -25,7 +25,7 @@
 #include <variant>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 // clang-format off
 #include "cutlass/cutlass.h"
diff --git a/src/runtime/contrib/cutlass/weight_preprocess.cc b/src/runtime/extra/contrib/cutlass/weight_preprocess.cc
similarity index 100%
rename from src/runtime/contrib/cutlass/weight_preprocess.cc
rename to src/runtime/extra/contrib/cutlass/weight_preprocess.cc
diff --git a/src/runtime/contrib/dnnl/dnnl.cc b/src/runtime/extra/contrib/dnnl/dnnl.cc
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl.cc
rename to src/runtime/extra/contrib/dnnl/dnnl.cc
diff --git a/src/runtime/contrib/dnnl/dnnl_json_runtime.cc b/src/runtime/extra/contrib/dnnl/dnnl_json_runtime.cc
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl_json_runtime.cc
rename to src/runtime/extra/contrib/dnnl/dnnl_json_runtime.cc
diff --git a/src/runtime/contrib/dnnl/dnnl_kernel.h b/src/runtime/extra/contrib/dnnl/dnnl_kernel.h
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl_kernel.h
rename to src/runtime/extra/contrib/dnnl/dnnl_kernel.h
diff --git a/src/runtime/contrib/dnnl/dnnl_tensor_requisite.h b/src/runtime/extra/contrib/dnnl/dnnl_tensor_requisite.h
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl_tensor_requisite.h
rename to src/runtime/extra/contrib/dnnl/dnnl_tensor_requisite.h
diff --git a/src/runtime/contrib/dnnl/dnnl_utils.cc b/src/runtime/extra/contrib/dnnl/dnnl_utils.cc
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl_utils.cc
rename to src/runtime/extra/contrib/dnnl/dnnl_utils.cc
diff --git a/src/runtime/contrib/dnnl/dnnl_utils.h b/src/runtime/extra/contrib/dnnl/dnnl_utils.h
similarity index 100%
rename from src/runtime/contrib/dnnl/dnnl_utils.h
rename to src/runtime/extra/contrib/dnnl/dnnl_utils.h
diff --git a/src/runtime/contrib/example_npu/example_npu_runtime.cc b/src/runtime/extra/contrib/example_npu/example_npu_runtime.cc
similarity index 100%
rename from src/runtime/contrib/example_npu/example_npu_runtime.cc
rename to src/runtime/extra/contrib/example_npu/example_npu_runtime.cc
diff --git a/src/runtime/contrib/hipblas/hipblas.cc b/src/runtime/extra/contrib/hipblas/hipblas.cc
similarity index 99%
rename from src/runtime/contrib/hipblas/hipblas.cc
rename to src/runtime/extra/contrib/hipblas/hipblas.cc
index eca971e06606..99fc3cd66639 100644
--- a/src/runtime/contrib/hipblas/hipblas.cc
+++ b/src/runtime/extra/contrib/hipblas/hipblas.cc
@@ -25,7 +25,7 @@
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/runtime/data_type.h>
 
-#include "../../3rdparty/compiler-rt/builtin_fp16.h"
+#include "../../../../../3rdparty/compiler-rt/builtin_fp16.h"
 #include "../cblas/gemm_common.h"
 #include "hipblas_utils.h"
 
diff --git a/src/runtime/contrib/hipblas/hipblas_json_runtime.cc b/src/runtime/extra/contrib/hipblas/hipblas_json_runtime.cc
similarity index 99%
rename from src/runtime/contrib/hipblas/hipblas_json_runtime.cc
rename to src/runtime/extra/contrib/hipblas/hipblas_json_runtime.cc
index 6f5708eb4922..f352e184f426 100644
--- a/src/runtime/contrib/hipblas/hipblas_json_runtime.cc
+++ b/src/runtime/extra/contrib/hipblas/hipblas_json_runtime.cc
@@ -32,7 +32,7 @@
 #include <string>
 #include <vector>
 
-#include "../../rocm/rocm_common.h"
+#include "../../../rocm/rocm_common.h"
 #include "../json/json_node.h"
 #include "../json/json_runtime.h"
 #include "hipblas_utils.h"
diff --git a/src/runtime/contrib/hipblas/hipblas_utils.cc b/src/runtime/extra/contrib/hipblas/hipblas_utils.cc
similarity index 98%
rename from src/runtime/contrib/hipblas/hipblas_utils.cc
rename to src/runtime/extra/contrib/hipblas/hipblas_utils.cc
index a7de7310ba13..2ea815c676e9 100644
--- a/src/runtime/contrib/hipblas/hipblas_utils.cc
+++ b/src/runtime/extra/contrib/hipblas/hipblas_utils.cc
@@ -25,7 +25,7 @@
 #include <tvm/ffi/extra/c_env_api.h>
 #include <tvm/ffi/function.h>
 
-#include "../../rocm/rocm_common.h"
+#include "../../../rocm/rocm_common.h"
 
 namespace tvm {
 namespace contrib {
diff --git a/src/runtime/contrib/hipblas/hipblas_utils.h b/src/runtime/extra/contrib/hipblas/hipblas_utils.h
similarity index 100%
rename from src/runtime/contrib/hipblas/hipblas_utils.h
rename to src/runtime/extra/contrib/hipblas/hipblas_utils.h
diff --git a/src/runtime/contrib/json/json_node.h b/src/runtime/extra/contrib/json/json_node.h
similarity index 100%
rename from src/runtime/contrib/json/json_node.h
rename to src/runtime/extra/contrib/json/json_node.h
diff --git a/src/runtime/contrib/json/json_runtime.h b/src/runtime/extra/contrib/json/json_runtime.h
similarity index 99%
rename from src/runtime/contrib/json/json_runtime.h
rename to src/runtime/extra/contrib/json/json_runtime.h
index 27fa76bd0aed..bc116372686a 100644
--- a/src/runtime/contrib/json/json_runtime.h
+++ b/src/runtime/extra/contrib/json/json_runtime.h
@@ -40,7 +40,7 @@
 #include <utility>
 #include <vector>
 
-#include "../../../support/bytes_io.h"
+#include "../../../../support/bytes_io.h"
 #include "json_node.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/nnapi/nnapi_builder.cc b/src/runtime/extra/contrib/nnapi/nnapi_builder.cc
similarity index 100%
rename from src/runtime/contrib/nnapi/nnapi_builder.cc
rename to src/runtime/extra/contrib/nnapi/nnapi_builder.cc
diff --git a/src/runtime/contrib/nnapi/nnapi_builder.h b/src/runtime/extra/contrib/nnapi/nnapi_builder.h
similarity index 100%
rename from src/runtime/contrib/nnapi/nnapi_builder.h
rename to src/runtime/extra/contrib/nnapi/nnapi_builder.h
diff --git a/src/runtime/contrib/nnapi/nnapi_ops.cc b/src/runtime/extra/contrib/nnapi/nnapi_ops.cc
similarity index 100%
rename from src/runtime/contrib/nnapi/nnapi_ops.cc
rename to src/runtime/extra/contrib/nnapi/nnapi_ops.cc
diff --git a/src/runtime/contrib/nnapi/nnapi_ops.h b/src/runtime/extra/contrib/nnapi/nnapi_ops.h
similarity index 100%
rename from src/runtime/contrib/nnapi/nnapi_ops.h
rename to src/runtime/extra/contrib/nnapi/nnapi_ops.h
diff --git a/src/runtime/contrib/nnapi/nnapi_runtime.cc b/src/runtime/extra/contrib/nnapi/nnapi_runtime.cc
similarity index 100%
rename from src/runtime/contrib/nnapi/nnapi_runtime.cc
rename to src/runtime/extra/contrib/nnapi/nnapi_runtime.cc
diff --git a/src/runtime/contrib/nvshmem/dist_gemm.cu b/src/runtime/extra/contrib/nvshmem/dist_gemm.cu
similarity index 99%
rename from src/runtime/contrib/nvshmem/dist_gemm.cu
rename to src/runtime/extra/contrib/nvshmem/dist_gemm.cu
index e4b8a1afe3af..512613f2ed5e 100644
--- a/src/runtime/contrib/nvshmem/dist_gemm.cu
+++ b/src/runtime/extra/contrib/nvshmem/dist_gemm.cu
@@ -23,7 +23,7 @@
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/runtime/disco/disco_worker.h>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/contrib/nvshmem/init.cc b/src/runtime/extra/contrib/nvshmem/init.cc
similarity index 99%
rename from src/runtime/contrib/nvshmem/init.cc
rename to src/runtime/extra/contrib/nvshmem/init.cc
index a69703949605..b25390a613f6 100644
--- a/src/runtime/contrib/nvshmem/init.cc
+++ b/src/runtime/extra/contrib/nvshmem/init.cc
@@ -26,7 +26,7 @@
 #include <tvm/runtime/disco/disco_worker.h>
 #include <tvm/runtime/logging.h>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/contrib/nvshmem/kv_transfer.cu b/src/runtime/extra/contrib/nvshmem/kv_transfer.cu
similarity index 100%
rename from src/runtime/contrib/nvshmem/kv_transfer.cu
rename to src/runtime/extra/contrib/nvshmem/kv_transfer.cu
diff --git a/src/runtime/contrib/nvshmem/memory_allocator.cc b/src/runtime/extra/contrib/nvshmem/memory_allocator.cc
similarity index 97%
rename from src/runtime/contrib/nvshmem/memory_allocator.cc
rename to src/runtime/extra/contrib/nvshmem/memory_allocator.cc
index 325f535be620..e1806e4c4b95 100644
--- a/src/runtime/contrib/nvshmem/memory_allocator.cc
+++ b/src/runtime/extra/contrib/nvshmem/memory_allocator.cc
@@ -24,9 +24,9 @@
 
 #include <thread>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
+#include "../../../memory/pooled_allocator.h"
 #include "../../disco/utils.h"
-#include "../../memory/pooled_allocator.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/contrib/random/mt_random_engine.cc b/src/runtime/extra/contrib/random/mt_random_engine.cc
similarity index 100%
rename from src/runtime/contrib/random/mt_random_engine.cc
rename to src/runtime/extra/contrib/random/mt_random_engine.cc
diff --git a/src/runtime/contrib/random/random.cc b/src/runtime/extra/contrib/random/random.cc
similarity index 100%
rename from src/runtime/contrib/random/random.cc
rename to src/runtime/extra/contrib/random/random.cc
diff --git a/src/runtime/contrib/sort/sort.cc b/src/runtime/extra/contrib/sort/sort.cc
similarity index 99%
rename from src/runtime/contrib/sort/sort.cc
rename to src/runtime/extra/contrib/sort/sort.cc
index 0d072c963846..e97fa8e76367 100644
--- a/src/runtime/contrib/sort/sort.cc
+++ b/src/runtime/extra/contrib/sort/sort.cc
@@ -30,7 +30,7 @@
 #include <algorithm>
 #include <vector>
 
-#include "../../../../3rdparty/compiler-rt/builtin_fp16.h"
+#include "../../../../../3rdparty/compiler-rt/builtin_fp16.h"
 
 namespace tvm {
 namespace contrib {
diff --git a/src/runtime/contrib/tensorrt/tensorrt_builder.cc b/src/runtime/extra/contrib/tensorrt/tensorrt_builder.cc
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_builder.cc
rename to src/runtime/extra/contrib/tensorrt/tensorrt_builder.cc
diff --git a/src/runtime/contrib/tensorrt/tensorrt_builder.h b/src/runtime/extra/contrib/tensorrt/tensorrt_builder.h
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_builder.h
rename to src/runtime/extra/contrib/tensorrt/tensorrt_builder.h
diff --git a/src/runtime/contrib/tensorrt/tensorrt_calibrator.h b/src/runtime/extra/contrib/tensorrt/tensorrt_calibrator.h
similarity index 99%
rename from src/runtime/contrib/tensorrt/tensorrt_calibrator.h
rename to src/runtime/extra/contrib/tensorrt/tensorrt_calibrator.h
index fea1a4684df4..d9e8df9d38e1 100755
--- a/src/runtime/contrib/tensorrt/tensorrt_calibrator.h
+++ b/src/runtime/extra/contrib/tensorrt/tensorrt_calibrator.h
@@ -26,7 +26,7 @@
 #include <string>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 #include "NvInfer.h"
 
 namespace tvm {
diff --git a/src/runtime/contrib/tensorrt/tensorrt_logger.h b/src/runtime/extra/contrib/tensorrt/tensorrt_logger.h
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_logger.h
rename to src/runtime/extra/contrib/tensorrt/tensorrt_logger.h
diff --git a/src/runtime/contrib/tensorrt/tensorrt_ops.cc b/src/runtime/extra/contrib/tensorrt/tensorrt_ops.cc
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_ops.cc
rename to src/runtime/extra/contrib/tensorrt/tensorrt_ops.cc
diff --git a/src/runtime/contrib/tensorrt/tensorrt_ops.h b/src/runtime/extra/contrib/tensorrt/tensorrt_ops.h
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_ops.h
rename to src/runtime/extra/contrib/tensorrt/tensorrt_ops.h
diff --git a/src/runtime/contrib/tensorrt/tensorrt_runtime.cc b/src/runtime/extra/contrib/tensorrt/tensorrt_runtime.cc
similarity index 99%
rename from src/runtime/contrib/tensorrt/tensorrt_runtime.cc
rename to src/runtime/extra/contrib/tensorrt/tensorrt_runtime.cc
index d4fcffd541bb..40ca760d96f2 100644
--- a/src/runtime/contrib/tensorrt/tensorrt_runtime.cc
+++ b/src/runtime/extra/contrib/tensorrt/tensorrt_runtime.cc
@@ -34,8 +34,8 @@
 #include <unordered_map>
 #include <vector>
 
-#include "../../../support/env.h"
-#include "../../file_utils.h"
+#include "../../../../support/env.h"
+#include "../../../file_utils.h"
 #include "../json/json_node.h"
 #include "../json/json_runtime.h"
 
diff --git a/src/runtime/contrib/tensorrt/tensorrt_utils.h b/src/runtime/extra/contrib/tensorrt/tensorrt_utils.h
similarity index 100%
rename from src/runtime/contrib/tensorrt/tensorrt_utils.h
rename to src/runtime/extra/contrib/tensorrt/tensorrt_utils.h
diff --git a/src/runtime/contrib/thrust/thrust.cu b/src/runtime/extra/contrib/thrust/thrust.cu
similarity index 99%
rename from src/runtime/contrib/thrust/thrust.cu
rename to src/runtime/extra/contrib/thrust/thrust.cu
index 16217432dc98..7c3930f0c81b 100644
--- a/src/runtime/contrib/thrust/thrust.cu
+++ b/src/runtime/extra/contrib/thrust/thrust.cu
@@ -42,7 +42,7 @@
 #include <memory>
 #include <vector>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 namespace tvm {
 namespace contrib {
 
diff --git a/src/runtime/contrib/vllm/attention_kernels.cu b/src/runtime/extra/contrib/vllm/attention_kernels.cu
similarity index 100%
rename from src/runtime/contrib/vllm/attention_kernels.cu
rename to src/runtime/extra/contrib/vllm/attention_kernels.cu
diff --git a/src/runtime/contrib/vllm/attention_utils.cuh b/src/runtime/extra/contrib/vllm/attention_utils.cuh
similarity index 100%
rename from src/runtime/contrib/vllm/attention_utils.cuh
rename to src/runtime/extra/contrib/vllm/attention_utils.cuh
diff --git a/src/runtime/contrib/vllm/cache_alloc.cc b/src/runtime/extra/contrib/vllm/cache_alloc.cc
similarity index 100%
rename from src/runtime/contrib/vllm/cache_alloc.cc
rename to src/runtime/extra/contrib/vllm/cache_alloc.cc
diff --git a/src/runtime/contrib/vllm/cache_kernels.cu b/src/runtime/extra/contrib/vllm/cache_kernels.cu
similarity index 100%
rename from src/runtime/contrib/vllm/cache_kernels.cu
rename to src/runtime/extra/contrib/vllm/cache_kernels.cu
diff --git a/src/runtime/contrib/vllm/dtype_float16.h b/src/runtime/extra/contrib/vllm/dtype_float16.h
similarity index 100%
rename from src/runtime/contrib/vllm/dtype_float16.h
rename to src/runtime/extra/contrib/vllm/dtype_float16.h
diff --git a/src/runtime/disco/bcast_session.cc b/src/runtime/extra/disco/bcast_session.cc
similarity index 100%
rename from src/runtime/disco/bcast_session.cc
rename to src/runtime/extra/disco/bcast_session.cc
diff --git a/src/runtime/disco/bcast_session.h b/src/runtime/extra/disco/bcast_session.h
similarity index 100%
rename from src/runtime/disco/bcast_session.h
rename to src/runtime/extra/disco/bcast_session.h
diff --git a/src/runtime/disco/builtin.cc b/src/runtime/extra/disco/builtin.cc
similarity index 100%
rename from src/runtime/disco/builtin.cc
rename to src/runtime/extra/disco/builtin.cc
diff --git a/src/runtime/disco/cuda_ipc/cuda_ipc_memory.cc b/src/runtime/extra/disco/cuda_ipc/cuda_ipc_memory.cc
similarity index 98%
rename from src/runtime/disco/cuda_ipc/cuda_ipc_memory.cc
rename to src/runtime/extra/disco/cuda_ipc/cuda_ipc_memory.cc
index fcc92badf334..c83cba280ab7 100644
--- a/src/runtime/disco/cuda_ipc/cuda_ipc_memory.cc
+++ b/src/runtime/extra/disco/cuda_ipc/cuda_ipc_memory.cc
@@ -23,9 +23,9 @@
 #include <tvm/runtime/disco/cuda_ipc_memory.h>
 #include <tvm/runtime/memory/memory_manager.h>
 
-#include "../../../../3rdparty/tensorrt_llm/custom_allreduce_kernels.h"
-#include "../../cuda/cuda_common.h"
-#include "../../memory/pooled_allocator.h"
+#include "../../../../../3rdparty/tensorrt_llm/custom_allreduce_kernels.h"
+#include "../../../cuda/cuda_common.h"
+#include "../../../memory/pooled_allocator.h"
 #include "../nccl/nccl_context.h"
 
 namespace tvm {
diff --git a/src/runtime/disco/cuda_ipc/custom_allreduce.cc b/src/runtime/extra/disco/cuda_ipc/custom_allreduce.cc
similarity index 98%
rename from src/runtime/disco/cuda_ipc/custom_allreduce.cc
rename to src/runtime/extra/disco/cuda_ipc/custom_allreduce.cc
index 69652c7e82c9..ffe00d5feef9 100644
--- a/src/runtime/disco/cuda_ipc/custom_allreduce.cc
+++ b/src/runtime/extra/disco/cuda_ipc/custom_allreduce.cc
@@ -23,7 +23,7 @@
 #include <tvm/runtime/disco/cuda_ipc_memory.h>
 #include <tvm/runtime/memory/memory_manager.h>
 
-#include "../../../../3rdparty/tensorrt_llm/custom_allreduce_kernels.h"
+#include "../../../../../3rdparty/tensorrt_llm/custom_allreduce_kernels.h"
 #include "../nccl/nccl_context.h"
 
 namespace tvm {
diff --git a/src/runtime/disco/disco_worker.cc b/src/runtime/extra/disco/disco_worker.cc
similarity index 99%
rename from src/runtime/disco/disco_worker.cc
rename to src/runtime/extra/disco/disco_worker.cc
index 24f5971fc791..5649dbac7943 100644
--- a/src/runtime/disco/disco_worker.cc
+++ b/src/runtime/extra/disco/disco_worker.cc
@@ -21,7 +21,7 @@
 #include <tvm/runtime/disco/disco_worker.h>
 #include <tvm/runtime/disco/session.h>
 
-#include "../../support/process_id.h"
+#include "../../../support/process_id.h"
 #include "./protocol.h"
 
 namespace tvm {
diff --git a/src/runtime/disco/disco_worker_thread.h b/src/runtime/extra/disco/disco_worker_thread.h
similarity index 100%
rename from src/runtime/disco/disco_worker_thread.h
rename to src/runtime/extra/disco/disco_worker_thread.h
diff --git a/src/runtime/disco/distributed/socket_session.cc b/src/runtime/extra/disco/distributed/socket_session.cc
similarity index 99%
rename from src/runtime/disco/distributed/socket_session.cc
rename to src/runtime/extra/disco/distributed/socket_session.cc
index a9d9d912aa82..3fb14914cab1 100644
--- a/src/runtime/disco/distributed/socket_session.cc
+++ b/src/runtime/extra/disco/distributed/socket_session.cc
@@ -22,7 +22,7 @@
 
 #include <numeric>
 
-#include "../../../support/socket.h"
+#include "../../../../support/socket.h"
 #include "../bcast_session.h"
 #include "../message_queue.h"
 
diff --git a/src/runtime/disco/loader.cc b/src/runtime/extra/disco/loader.cc
similarity index 99%
rename from src/runtime/disco/loader.cc
rename to src/runtime/extra/disco/loader.cc
index 891afa103ab4..1e7389541177 100644
--- a/src/runtime/disco/loader.cc
+++ b/src/runtime/extra/disco/loader.cc
@@ -29,7 +29,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "../file_utils.h"
+#include "../../file_utils.h"
 #include "./utils.h"
 
 namespace tvm {
diff --git a/src/runtime/disco/message_queue.h b/src/runtime/extra/disco/message_queue.h
similarity index 100%
rename from src/runtime/disco/message_queue.h
rename to src/runtime/extra/disco/message_queue.h
diff --git a/src/runtime/disco/nccl/nccl.cc b/src/runtime/extra/disco/nccl/nccl.cc
similarity index 99%
rename from src/runtime/disco/nccl/nccl.cc
rename to src/runtime/extra/disco/nccl/nccl.cc
index 3167ab243ca7..887f440b1b4f 100644
--- a/src/runtime/disco/nccl/nccl.cc
+++ b/src/runtime/extra/disco/nccl/nccl.cc
@@ -25,7 +25,7 @@
 #include <sstream>
 #include <vector>
 
-#include "../../../support/process_id.h"
+#include "../../../../support/process_id.h"
 #include "../utils.h"
 #include "nccl_context.h"
 
diff --git a/src/runtime/disco/nccl/nccl_context.h b/src/runtime/extra/disco/nccl/nccl_context.h
similarity index 97%
rename from src/runtime/disco/nccl/nccl_context.h
rename to src/runtime/extra/disco/nccl/nccl_context.h
index e18137f42d18..1434a7c4a2e1 100644
--- a/src/runtime/disco/nccl/nccl_context.h
+++ b/src/runtime/extra/disco/nccl/nccl_context.h
@@ -26,7 +26,7 @@
 #include <tvm/runtime/disco/builtin.h>
 #include <tvm/runtime/disco/session.h>
 
-#include "../../../support/process_id.h"
+#include "../../../../support/process_id.h"
 #include "../utils.h"
 
 /* `TVM_NCCL_RCCL_SWITCH` is set to 0 for NCCL, 1 for RCCL */
@@ -36,11 +36,11 @@
 #if TVM_NCCL_RCCL_SWITCH == 0
 #include <nccl.h>
 
-#include "../../cuda/cuda_common.h"
+#include "../../../cuda/cuda_common.h"
 #else
 #include <rccl/rccl.h>
 
-#include "../../rocm/rocm_common.h"
+#include "../../../rocm/rocm_common.h"
 #endif
 
 namespace tvm {
diff --git a/src/runtime/disco/process_session.cc b/src/runtime/extra/disco/process_session.cc
similarity index 98%
rename from src/runtime/disco/process_session.cc
rename to src/runtime/extra/disco/process_session.cc
index 8c976ee55e4f..41888caeced0 100644
--- a/src/runtime/disco/process_session.cc
+++ b/src/runtime/extra/disco/process_session.cc
@@ -26,8 +26,8 @@
 #include <utility>
 #include <vector>
 
-#include "../../support/pipe.h"
-#include "../minrpc/rpc_reference.h"
+#include "../../../support/pipe.h"
+#include "../../rpc/minrpc/rpc_reference.h"
 #include "./bcast_session.h"
 #include "./disco_worker_thread.h"
 #include "./message_queue.h"
diff --git a/src/runtime/disco/protocol.h b/src/runtime/extra/disco/protocol.h
similarity index 98%
rename from src/runtime/disco/protocol.h
rename to src/runtime/extra/disco/protocol.h
index bf3661292b3a..89905b7ce117 100644
--- a/src/runtime/disco/protocol.h
+++ b/src/runtime/extra/disco/protocol.h
@@ -30,10 +30,10 @@
 #include <utility>
 #include <vector>
 
-#include "../../support/arena.h"
-#include "../../support/base64.h"
-#include "../../support/bytes_io.h"
-#include "../minrpc/rpc_reference.h"
+#include "../../../support/arena.h"
+#include "../../../support/base64.h"
+#include "../../../support/bytes_io.h"
+#include "../../rpc/minrpc/rpc_reference.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/disco/session.cc b/src/runtime/extra/disco/session.cc
similarity index 100%
rename from src/runtime/disco/session.cc
rename to src/runtime/extra/disco/session.cc
diff --git a/src/runtime/disco/threaded_session.cc b/src/runtime/extra/disco/threaded_session.cc
similarity index 98%
rename from src/runtime/disco/threaded_session.cc
rename to src/runtime/extra/disco/threaded_session.cc
index ddd767168051..83de74766bb7 100644
--- a/src/runtime/disco/threaded_session.cc
+++ b/src/runtime/extra/disco/threaded_session.cc
@@ -28,8 +28,8 @@
 #include <thread>
 #include <utility>
 
-#include "../../support/ring_buffer.h"
-#include "../minrpc/rpc_reference.h"
+#include "../../../support/ring_buffer.h"
+#include "../../rpc/minrpc/rpc_reference.h"
 #include "./bcast_session.h"
 #include "./disco_worker_thread.h"
 #include "./protocol.h"
diff --git a/src/runtime/disco/utils.h b/src/runtime/extra/disco/utils.h
similarity index 100%
rename from src/runtime/disco/utils.h
rename to src/runtime/extra/disco/utils.h
diff --git a/src/runtime/hexagon/rpc/hexagon/rpc_server.cc b/src/runtime/hexagon/rpc/hexagon/rpc_server.cc
index 9f20a8f6d229..40dc3f34b73d 100644
--- a/src/runtime/hexagon/rpc/hexagon/rpc_server.cc
+++ b/src/runtime/hexagon/rpc/hexagon/rpc_server.cc
@@ -36,7 +36,7 @@ extern "C" {
 #include <memory>
 #include <string>
 
-#include "../../../minrpc/minrpc_server.h"
+#include "../../../rpc/minrpc/minrpc_server.h"
 #include "../../hexagon/hexagon_common.h"
 #include "../../hexagon/hexagon_device_api.h"
 #include "../../profiler/prof_utils.h"
diff --git a/src/runtime/hexagon/rpc/simulator/rpc_server.cc b/src/runtime/hexagon/rpc/simulator/rpc_server.cc
index 2cef7f9c712e..61bd055dc4fe 100644
--- a/src/runtime/hexagon/rpc/simulator/rpc_server.cc
+++ b/src/runtime/hexagon/rpc/simulator/rpc_server.cc
@@ -28,7 +28,7 @@
 #include <sstream>
 #include <string>
 
-#include "../../../minrpc/minrpc_server.h"
+#include "../../../rpc/minrpc/minrpc_server.h"
 #include "../../hexagon_common.h"
 #include "../../profiler/prof_utils.h"
 #include "hexagon_sim_proto.h"
diff --git a/src/runtime/opencl/opencl_common.h b/src/runtime/opencl/opencl_common.h
index df2f370fd038..7b9a76dc3a8e 100644
--- a/src/runtime/opencl/opencl_common.h
+++ b/src/runtime/opencl/opencl_common.h
@@ -73,8 +73,8 @@
 #include "../file_utils.h"
 #include "../metadata.h"
 #include "../pack_args.h"
-#include "../texture.h"
 #include "../thread_storage_scope.h"
+#include "texture.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/texture.h b/src/runtime/opencl/texture.h
similarity index 97%
rename from src/runtime/texture.h
rename to src/runtime/opencl/texture.h
index f8ed6cf38adf..a8711805cbfa 100644
--- a/src/runtime/texture.h
+++ b/src/runtime/opencl/texture.h
@@ -21,8 +21,8 @@
  * \file texture.h
  * \brief Texture utilities
  */
-#ifndef TVM_RUNTIME_TEXTURE_H_
-#define TVM_RUNTIME_TEXTURE_H_
+#ifndef TVM_RUNTIME_OPENCL_TEXTURE_H_
+#define TVM_RUNTIME_OPENCL_TEXTURE_H_
 
 #include <tvm/runtime/device_api.h>
 
@@ -135,4 +135,4 @@ inline DataType GetChannelType(size_t channel_size) {
 
 }  // namespace runtime
 }  // namespace tvm
-#endif  // TVM_RUNTIME_TEXTURE_H_
+#endif  // TVM_RUNTIME_OPENCL_TEXTURE_H_
diff --git a/src/runtime/minrpc/minrpc_server.h b/src/runtime/rpc/minrpc/minrpc_server.h
similarity index 100%
rename from src/runtime/minrpc/minrpc_server.h
rename to src/runtime/rpc/minrpc/minrpc_server.h
diff --git a/src/runtime/minrpc/posix_popen_server/posix_popen_server.cc b/src/runtime/rpc/minrpc/posix_popen_server/posix_popen_server.cc
similarity index 100%
rename from src/runtime/minrpc/posix_popen_server/posix_popen_server.cc
rename to src/runtime/rpc/minrpc/posix_popen_server/posix_popen_server.cc
diff --git a/src/runtime/minrpc/rpc_reference.h b/src/runtime/rpc/minrpc/rpc_reference.h
similarity index 100%
rename from src/runtime/minrpc/rpc_reference.h
rename to src/runtime/rpc/minrpc/rpc_reference.h
diff --git a/src/runtime/rpc/rpc_endpoint.h b/src/runtime/rpc/rpc_endpoint.h
index 9438470cb215..6f713fb1288d 100644
--- a/src/runtime/rpc/rpc_endpoint.h
+++ b/src/runtime/rpc/rpc_endpoint.h
@@ -32,7 +32,7 @@
 #include <utility>
 
 #include "../../support/ring_buffer.h"
-#include "../minrpc/rpc_reference.h"
+#include "minrpc/rpc_reference.h"
 #include "rpc_channel.h"
 #include "rpc_session.h"
 
diff --git a/src/runtime/rpc/rpc_session.h b/src/runtime/rpc/rpc_session.h
index d694173b8de7..1276d8e267a1 100644
--- a/src/runtime/rpc/rpc_session.h
+++ b/src/runtime/rpc/rpc_session.h
@@ -32,7 +32,7 @@
 #include <memory>
 #include <string>
 
-#include "../minrpc/rpc_reference.h"
+#include "minrpc/rpc_reference.h"
 
 namespace tvm {
 namespace runtime {
diff --git a/src/runtime/thread_map.h b/src/runtime/vulkan/thread_map.h
similarity index 97%
rename from src/runtime/thread_map.h
rename to src/runtime/vulkan/thread_map.h
index c3fc7e31e9bd..2b10b299fe6c 100644
--- a/src/runtime/thread_map.h
+++ b/src/runtime/vulkan/thread_map.h
@@ -17,8 +17,8 @@
  * under the License.
  */
 
-#ifndef TVM_RUNTIME_THREAD_MAP_H_
-#define TVM_RUNTIME_THREAD_MAP_H_
+#ifndef TVM_RUNTIME_VULKAN_THREAD_MAP_H_
+#define TVM_RUNTIME_VULKAN_THREAD_MAP_H_
 
 #include <functional>
 #include <memory>
@@ -172,4 +172,4 @@ class ThreadMap {
 }  // namespace runtime
 }  // namespace tvm
 
-#endif  // TVM_RUNTIME_THREAD_MAP_H_
+#endif  // TVM_RUNTIME_VULKAN_THREAD_MAP_H_
diff --git a/src/runtime/vulkan/vulkan_device.h b/src/runtime/vulkan/vulkan_device.h
index c327149cc2b0..7e75f4eb3ed3 100644
--- a/src/runtime/vulkan/vulkan_device.h
+++ b/src/runtime/vulkan/vulkan_device.h
@@ -30,7 +30,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "../thread_map.h"
+#include "thread_map.h"
 #include "vulkan/vulkan_core.h"
 #include "vulkan_buffer.h"
 #include "vulkan_stream.h"
diff --git a/src/runtime/vulkan/vulkan_device_api.h b/src/runtime/vulkan/vulkan_device_api.h
index 5e9bfeb8c086..c39d5754d8cd 100644
--- a/src/runtime/vulkan/vulkan_device_api.h
+++ b/src/runtime/vulkan/vulkan_device_api.h
@@ -26,8 +26,8 @@
 #include <string>
 #include <vector>
 
-#include "../thread_map.h"
 #include "../workspace_pool.h"
+#include "thread_map.h"
 #include "vulkan/vulkan_core.h"
 #include "vulkan_device.h"
 #include "vulkan_instance.h"
diff --git a/src/s_tir/backend/adreno/inject_texture_alloc.cc b/src/s_tir/backend/adreno/inject_texture_alloc.cc
index 195168a35b31..f52a6d7148c6 100644
--- a/src/s_tir/backend/adreno/inject_texture_alloc.cc
+++ b/src/s_tir/backend/adreno/inject_texture_alloc.cc
@@ -27,7 +27,7 @@
 #include <tvm/tirx/stmt_functor.h>
 
 #include "../../../arith/ir_mutator_with_analyzer.h"
-#include "../../../runtime/texture.h"
+#include "../../../runtime/opencl/texture.h"
 #include "../../../tirx/transform/ir_utils.h"
 
 namespace tvm {
diff --git a/src/s_tir/backend/adreno/texture_flatten.cc b/src/s_tir/backend/adreno/texture_flatten.cc
index ca0c6b47204d..0ef074789652 100644
--- a/src/s_tir/backend/adreno/texture_flatten.cc
+++ b/src/s_tir/backend/adreno/texture_flatten.cc
@@ -33,7 +33,7 @@
 #include <unordered_map>
 
 #include "../../../arith/ir_visitor_with_analyzer.h"
-#include "../../../runtime/texture.h"
+#include "../../../runtime/opencl/texture.h"
 #include "../../../runtime/thread_storage_scope.h"
 
 namespace tvm {
diff --git a/src/target/opencl/codegen_opencl.cc b/src/target/opencl/codegen_opencl.cc
index 1b1eabe7ef4b..7016f0fbbf06 100644
--- a/src/target/opencl/codegen_opencl.cc
+++ b/src/target/opencl/codegen_opencl.cc
@@ -29,7 +29,7 @@
 #include <string>
 #include <vector>
 
-#include "../../runtime/texture.h"
+#include "../../runtime/opencl/texture.h"
 #include "../../runtime/thread_storage_scope.h"
 #include "../build_common.h"
 #include "opencl_fallback_module.h"
diff --git a/web/emcc/wasm_runtime.cc b/web/emcc/wasm_runtime.cc
index b2b9a470be7e..9d3d46f18cb4 100644
--- a/web/emcc/wasm_runtime.cc
+++ b/web/emcc/wasm_runtime.cc
@@ -32,9 +32,9 @@
 #include <tvm/ffi/reflection/registry.h>
 #include <tvm/runtime/logging.h>
 
-#include "src/runtime/contrib/sort/sort.cc"
 #include "src/runtime/cpu_device_api.cc"
 #include "src/runtime/device_api.cc"
+#include "src/runtime/extra/contrib/sort/sort.cc"
 #include "src/runtime/file_utils.cc"
 #include "src/runtime/logging.cc"
 #include "src/runtime/rpc/rpc_channel.cc"