Merge branch 'main' of github.com:SlugLab/CXLMemSim

Author: vickiegpt

CMakeLists.txt

@@ -1,6 +1,9 @@
 cmake_minimum_required(VERSION 3.25.0)
 project(CXLMemSim VERSION 0.1.0)
 
+option(CXLMEMSIM_BUILD_MICROBENCHMARKS "Build microbenchmarks" ON)
+option(CXLMEMSIM_ENABLE_RDMA "Enable RDMA transport when libraries are available" ON)
+
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     add_compile_options (-fdiagnostics-color=always)
 elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
@@ -20,17 +23,42 @@ add_compile_definitions(SPDLOG_USE_STD_FORMAT)
 # Use header-only spdlog to avoid library mismatch
 add_compile_definitions(SPDLOG_HEADER_ONLY)
 
-add_subdirectory(microbench)
+if(CXLMEMSIM_BUILD_MICROBENCHMARKS)
+    add_subdirectory(microbench)
+endif()
 
 list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR})
 
 find_package(cxxopts REQUIRED)
 find_package(spdlog REQUIRED)
-file(GLOB_RECURSE SOURCE_FILES src/cxl*.cpp src/policy.cpp src/helper.cpp src/incore.cpp src/uncore.cpp src/perf.cpp src/monitor.cpp)
 
 execute_process(COMMAND uname -r OUTPUT_VARIABLE arch OUTPUT_STRIP_TRAILING_WHITESPACE)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -pthread")
-set(CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -latomic")
+
+find_library(ATOMIC_LIB atomic)
+find_library(RT_LIB rt)
+if(NOT ATOMIC_LIB)
+    set(ATOMIC_LIB "" CACHE STRING "libatomic path" FORCE)
+endif()
+if(NOT RT_LIB)
+    set(RT_LIB "" CACHE STRING "librt path" FORCE)
+endif()
+
+set(SOURCE_FILES
+    src/cxlcontroller.cpp
+    src/cxlendpoint.cpp
+    src/policy.cpp
+    src/helper.cpp
+)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    list(APPEND SOURCE_FILES
+        src/incore.cpp
+        src/uncore.cpp
+        src/perf.cpp
+        src/monitor.cpp
+    )
+endif()
 
 
 # Create a static library of CXL core functionality
@@ -44,31 +72,40 @@ set_target_properties(cxlmemsim_latency PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
-find_library(RDMACM_LIB rdmacm)
-find_library(IBVERBS_LIB ibverbs)
-message(STATUS "RDMA support enabled: rdmacm=${RDMACM_LIB} ibverbs=${IBVERBS_LIB}")
-add_compile_definitions(HAS_RDMA)
-set(RDMA_LIBS ${RDMACM_LIB} ${IBVERBS_LIB})
+set(RDMA_LIBS)
+if(CXLMEMSIM_ENABLE_RDMA AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    find_library(RDMACM_LIB rdmacm)
+    find_library(IBVERBS_LIB ibverbs)
+    if(RDMACM_LIB AND IBVERBS_LIB)
+        message(STATUS "RDMA support enabled: rdmacm=${RDMACM_LIB} ibverbs=${IBVERBS_LIB}")
+        add_compile_definitions(HAS_RDMA)
+        set(RDMA_LIBS ${RDMACM_LIB} ${IBVERBS_LIB})
+    else()
+        message(STATUS "RDMA support disabled: rdmacm or ibverbs not found")
+    endif()
+else()
+    message(STATUS "RDMA support disabled")
+endif()
 
 # Server library (shared between server executable and tests)
 set(SERVER_LIB_SOURCES src/shared_memory_manager.cc src/shm_communication.cpp src/distributed_server.cpp src/tcp_communication.cpp src/rdma_communication.cpp src/hdm_decoder.cpp src/coherency_engine.cpp)
 add_library(cxlmemsim_server_lib STATIC ${SERVER_LIB_SOURCES})
 target_include_directories(cxlmemsim_server_lib PUBLIC include src ${cxxopts_INCLUDE_DIR} ${spdlog_INCLUDE_DIR})
-target_link_libraries(cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only rt ${RDMA_LIBS})
+target_link_libraries(cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only ${RT_LIB} ${RDMA_LIBS})
 
 # Build server (uses server library + main)
 add_executable(cxlmemsim_server src/main_server.cc)
 target_include_directories(cxlmemsim_server PRIVATE include src ${cxxopts_INCLUDE_DIR} ${spdlog_INCLUDE_DIR})
-target_link_libraries(cxlmemsim_server cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only rt)
+target_link_libraries(cxlmemsim_server cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only ${RT_LIB} ${ATOMIC_LIB})
 
 # Distributed SHM test
 add_executable(test_distributed_shm tests/test_distributed_shm.cpp)
 target_include_directories(test_distributed_shm PRIVATE include src ${cxxopts_INCLUDE_DIR} ${spdlog_INCLUDE_DIR})
-target_link_libraries(test_distributed_shm cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only rt)
+target_link_libraries(test_distributed_shm cxlmemsim_server_lib cxlmemsim cxxopts::cxxopts spdlog::spdlog_header_only ${RT_LIB} ${ATOMIC_LIB})
 
 # Installation targets
 install(TARGETS cxlmemsim_server
         RUNTIME DESTINATION bin)
 
 install(FILES include/tcp_communication.h include/rdma_communication.h
-        DESTINATION include/cxlmemsim)
+        DESTINATION include/cxlmemsim)

README.md

@@ -322,6 +322,89 @@ libnvcuda.so.1
 libnvcuda.so
 ```
 
+## Packaged QEMU/Spack Flow
+
+For macOS and repeatable QEMU smoke tests, use the companion Spack environment from the Ocean Spack fork. That environment builds the x86_64 CXL-capable QEMU tree, builds `cxlmemsim_server`, installs launch scripts, and documents the runtime variables used by QEMU and the server.
+
+```bash
+git clone https://github.com/vickiegpt/spack.git ocean-spack
+cd ocean-spack
+source share/spack/setup-env.sh
+spack env activate ./share/spack/environments/cxlmemsim
+spack concretize -f
+spack install
+spack load cxlmemsim
+```
+
+The detailed usage document is installed in the Spack checkout at:
+
+```text
+share/spack/environments/cxlmemsim/README.md
+```
+
+After loading the package, download the guest kernel and disk image:
+
+```bash
+cxlmemsim-download-qemu-image
+```
+
+The default image directory is `CXL_QEMU_IMAGE_DIR`. To keep the raw guest disk at 4 GB:
+
+```bash
+qemu-img resize "$CXL_QEMU_IMAGE_DIR/qemu.img" 4G
+cp "$CXL_QEMU_IMAGE_DIR/qemu.img" "$CXL_QEMU_IMAGE_DIR/qemu1.img"
+```
+
+Launch the default guest:
+
+```bash
+qemu_launch_cxl.sh
+```
+
+Launch the second guest image:
+
+```bash
+qemu_launch_cxl1.sh
+```
+
+The launcher reads the following runtime variables:
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `CXL_TRANSPORT_MODE` | `shm` | QEMU transport mode: `shm` or `tcp`. |
+| `CXL_MEMSIM_HOST` | `127.0.0.1` | Local host for TCP mode. |
+| `CXL_MEMSIM_PORT` | `9999` | Local TCP server port. |
+| `CXL_PGAS_SHM` | `/cxlmemsim_pgas` | POSIX shared-memory object used by QEMU SHM mode. |
+| `CXL_MEMSIM_SERVER_BINARY` | package `bin/cxlmemsim_server` | Server binary started before QEMU. |
+| `CXL_MEMSIM_SERVER_AUTOSTART` | `auto` | Set to `1` to require server startup or `0` to disable it. |
+
+QEMU's `shm` transport uses the PGAS shared-memory protocol, so the packaged launcher maps `CXL_TRANSPORT_MODE=shm` to the server's `--comm-mode pgas-shm`.
+
+Shared-memory launch:
+
+```bash
+CXL_TRANSPORT_MODE=shm \
+CXL_PGAS_SHM=/cxlmemsim_pgas \
+qemu_launch_cxl.sh
+```
+
+TCP launch on the local port:
+
+```bash
+CXL_TRANSPORT_MODE=tcp \
+CXL_MEMSIM_HOST=127.0.0.1 \
+CXL_MEMSIM_PORT=9999 \
+qemu_launch_cxl.sh
+```
+
+Inside the guest, quick checks are:
+
+```bash
+lspci | grep -i cxl
+dmesg | grep -i cxl
+ls /sys/bus/cxl/devices
+```
+
 ## Running CXLMemSim Server
 
 Basic Type 3 server:

include/helper.h

@@ -12,21 +12,41 @@
 #ifndef CXLMEMSIM_HELPER_H
 #define CXLMEMSIM_HELPER_H
 
+#if defined(__linux__)
+#define CXLMEMSIM_HAS_LINUX_PERF 1
 #include "incore.h"
 #include "uncore.h"
+#include <linux/perf_event.h>
+#else
+#define CXLMEMSIM_HAS_LINUX_PERF 0
+struct perf_event_attr;
+#endif
+
+#if defined(__x86_64__) || defined(__i386__)
+#include <x86intrin.h>
+#else
+#include <atomic>
+#endif
+
+#include <array>
 #include <cstdint>
 #include <filesystem>
-#include <linux/perf_event.h>
 #include <ranges>
 #include <spdlog/spdlog.h>
+#include <string>
+#include <sys/types.h>
+#include <tuple>
 #include <vector>
-#include <x86intrin.h>
 
 #define PAGE_SIZE 4096
 #define DATA_SIZE PAGE_SIZE
 #define MMAP_SIZE (PAGE_SIZE + DATA_SIZE)
 
+#if defined(__x86_64__) || defined(__i386__)
 #define barrier() _mm_mfence()
+#else
+#define barrier() std::atomic_thread_fence(std::memory_order_seq_cst)
+#endif
 
 /* CPU Models */
 enum {
@@ -102,8 +122,10 @@ struct Elem {
 
 class PMUInfo {
 public:
+#if CXLMEMSIM_HAS_LINUX_PERF
     std::vector<Uncore> chas;
     std::vector<Incore> cpus;
+#endif
     Helper *helper;
     PMUInfo(pid_t pid, Helper *h, PerfConfig *perf_config);
     ~PMUInfo();
@@ -132,6 +154,8 @@ class Helper {
                             const std::vector<uint64_t> &perf_conf1, const std::vector<uint64_t> &perf_conf2);
 };
 
+#if CXLMEMSIM_HAS_LINUX_PERF
 long perf_event_open(perf_event_attr *event_attr, pid_t pid, int cpu, int group_fd, unsigned long flags);
+#endif
 
 #endif // CXLMEMSIM_HELPER_H

include/lbr.h

@@ -21,8 +21,18 @@
 
 #include "cxlcontroller.h"
 #include "helper.h"
+#include <cstdint>
+#if CXLMEMSIM_HAS_LINUX_PERF
 #include <linux/perf_event.h>
 #include <sys/mman.h>
+#else
+struct perf_event_header {
+    uint32_t type;
+    uint16_t misc;
+    uint16_t size;
+};
+struct perf_event_mmap_page;
+#endif
 class CXLController; // Forward declaration
 
 struct lbr {

include/pebs.h

@@ -15,7 +15,11 @@
 #include "helper.h"
 #include <cstdint>
 #include <cxlcontroller.h>
+#if CXLMEMSIM_HAS_LINUX_PERF
 #include <sys/mman.h>
+#else
+struct perf_event_mmap_page;
+#endif
 #include <sys/types.h>
 
 class PEBS {

src/cxlcontroller.cpp

@@ -11,7 +11,6 @@
 
 #include "cxlcontroller.h"
 #include "lbr.h"
-#include "monitor.h"
 #include "../include/distributed_server.h"
 
 void CXLController::insert_end_point(CXLMemExpander *end_point) { this->cur_expanders.emplace_back(end_point); }
@@ -674,4 +673,4 @@ double CXLController::calculate_distributed_latency(
     }
 
     return base_latency + network_latency + coherency_latency;
-}
+}