Use scikit build

Author: haochengxia

.github/workflows/python.yml

@@ -0,0 +1,38 @@
+name: Python
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Prepare
+        run: bash scripts/install_dependency.sh
+
+      - name: Build main libCacheSim project
+        run: |
+          cmake -G Ninja -B build
+          ninja -C build
+
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest
+
+      - name: Build libCacheSim-python
+        run: |
+          cd libCacheSim-python
+          pip install -e .
+
+      - name: Run tests
+        run: |
+          cd libCacheSim-python
+          pytest tests/

CMakeLists.txt

@@ -247,6 +247,8 @@ else()
     message(STATUS "Building without test")
 endif()
 
+# Export variables for scikit-build -> build/export_vars.cmake
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim-python/export)
 
 # libCacheSim unified library compilation and installation
 # Create a single library that combines all modular libraries

grep

@@ -0,0 +1,116 @@
+-- The C compiler identification is GNU 11.4.0
+-- The CXX compiler identification is GNU 11.4.0
+-- Detecting C compiler ABI info
+-- Detecting C compiler ABI info - done
+-- Check for working C compiler: /usr/bin/cc - skipped
+-- Detecting C compile features
+-- Detecting C compile features - done
+-- Detecting CXX compiler ABI info
+-- Detecting CXX compiler ABI info - done
+-- Check for working CXX compiler: /usr/bin/c++ - skipped
+-- Detecting CXX compile features
+-- Detecting CXX compile features - done
+-- Version from /users/Haocheng/mylcs/version.txt: 0.3.1
+-- LOG_LEVEL is not set, use INFO as default for release build
+-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
+-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success
+-- Found Threads: TRUE
+-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2")
+-- Checking for one of the modules 'glib-2.0'
+-- Found GLib: /usr/lib/x86_64-linux-gnu/libglib-2.0.so (found version "2.72.4")
+-- Looking for argp_parse
+-- Looking for argp_parse - found
+-- Found argp: /usr/include
+-- Found ZSTD: /usr/local/lib/libzstd.so
+-- Found Zstd: /usr/local/lib/libzstd.so
+-- ZSTD_INCLUDE_DIR /usr/local/include, ZSTD_LIBRARIES /usr/local/lib/libzstd.so
+-- CMAKE_CXX_FLAGS_DEBUG -g
+-- CMAKE_CXX_FLAGS_RELWITHDEBINFO -O2 -g -DNDEBUG
+-- CMAKE_CXX_FLAGS_RELEASE -O3 -DNDEBUG
+-- SUPPORT TTL OFF, USE_HUGEPAGE ON
+-- LOGLEVEL default, ENABLE_GLCACHE OFF
+-- ENABLE_LRB OFF, ENABLE_3L_CACHE OFF
+-- OPT_SUPPORT_ZSTD_TRACE ON
+-- <<++=====------------------\/------------------=====++>>
+-- <<++              libCacheSim summary             ++>>
+-- <<++=====------------------/\------------------=====++>>
+-- ================== dependency related ==================
+-- glib found?     TRUE - LIBS=/usr/lib/x86_64-linux-gnu/libglib-2.0.so, header=/usr/include/glib-2.0;/usr/lib/x86_64-linux-gnu/glib-2.0/include
+-- tcmalloc found? TRUE - LIBS=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so, header=
+-- ZSTD found? TRUE - LIBS=/usr/local/lib/libzstd.so, header=/usr/local/include
+-- ==================== CMake related =====================
+-- platform          = Linux 5.15.0-138-generic
+-- CPU type          = x86_64
+-- cmake source      = /users/Haocheng/mylcs
+-- cmake compiler    = /usr/bin/cc
+-- CMAKE_BUILD_TYPE  = Release
+-- CFLAGS            = 
+-- dependency_libs    = /usr/lib/x86_64-linux-gnu/libglib-2.0.so;/usr/local/lib/libzstd.so;/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so
+-- Installation path = /usr/local
+-- ========================================================
+-- ============= Status of optional features ==============
+-- ========================================================
+-- cache_sources_c = admission/bloomfilter.c;admission/prob.c;admission/size.c;admission/sizeProbabilistic.c;eviction/ARC.c;eviction/ARCv0.c;eviction/Belady.c;eviction/BeladySize.c;eviction/CAR.c;eviction/Cacheus.c;eviction/Clock.c;eviction/ClockPro.c;eviction/FIFO.c;eviction/Hyperbolic.c;eviction/LeCaR.c;eviction/LeCaRv0.c;eviction/LFU.c;eviction/LFUDA.c;eviction/LIRS.c;eviction/LRU.c;eviction/LRUProb.c;eviction/LRUv0.c;eviction/MRU.c;eviction/nop.c;eviction/QDLP.c;eviction/Random.c;eviction/RandomLRU.c;eviction/RandomTwo.c;eviction/S3FIFO.c;eviction/Sieve.c;eviction/Size.c;eviction/SLRU.c;eviction/SLRUv0.c;eviction/TwoQ.c;eviction/WTinyLFU.c;eviction/CR_LFU.c;eviction/SR_LRU.c;eviction/FIFO_Merge.c;eviction/FIFO_Reinsertion.c;eviction/S3FIFOv0.c;eviction/S3FIFOd.c;eviction/other/flashProb.c;eviction/other/S3LRU.c;prefetch/Mithril.c;prefetch/OBL.c;prefetch/PG.c;cache.c;plugin.c
+-- cache_sources_cpp = admission/adaptsize/adaptsize.cpp;admission/adaptsize/adaptsize_interface.cpp;eviction/cpp/LFU.cpp;eviction/cpp/GDSF.cpp;eviction/LHD/lhd.cpp;eviction/LHD/LHD_Interface.cpp
+-- Building with test
+-- Exported variables to /users/Haocheng/mylcs/build/export_vars.cmake
+-- Configuring done (0.8s)
+-- Generating done (0.1s)
+-- Build files have been written to: /users/Haocheng/mylcs/build
+~/mylcs/libCacheSim-python ~/mylcs
+Using pip 22.0.2 from /usr/lib/python3/dist-packages/pip (python 3.10)
+Defaulting to user installation because normal site-packages is not writeable
+Created temporary directory: /tmp/pip-ephem-wheel-cache-b2vd6qte
+Created temporary directory: /tmp/pip-req-tracker-g5ne9qc8
+Initialized build tracking at /tmp/pip-req-tracker-g5ne9qc8
+Created build tracker: /tmp/pip-req-tracker-g5ne9qc8
+Entered build tracker: /tmp/pip-req-tracker-g5ne9qc8
+Created temporary directory: /tmp/pip-install-m8gstwzc
+Obtaining file:///users/Haocheng/mylcs/libCacheSim-python
+  Added file:///users/Haocheng/mylcs/libCacheSim-python to build tracker '/tmp/pip-req-tracker-g5ne9qc8'
+  Created temporary directory: /tmp/pip-build-env-55hvaxmg
+  Created temporary directory: /tmp/pip-standalone-pip-2s9ixk7n
+  Installing build dependencies: started
+  Installing build dependencies: finished with status 'canceled'
+Exception information:
+Traceback (most recent call last):
+  File "/usr/lib/python3/dist-packages/pip/_internal/cli/base_command.py", line 165, in exc_logging_wrapper
+    status = run_func(*args)
+  File "/usr/lib/python3/dist-packages/pip/_internal/cli/req_command.py", line 205, in wrapper
+    return func(self, options, args)
+  File "/usr/lib/python3/dist-packages/pip/_internal/commands/install.py", line 339, in run
+    requirement_set = resolver.resolve(
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/resolver.py", line 75, in resolve
+    collected = self.factory.collect_root_requirements(root_reqs)
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/factory.py", line 506, in collect_root_requirements
+    req = self._make_requirement_from_install_req(
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/factory.py", line 468, in _make_requirement_from_install_req
+    cand = self._make_candidate_from_link(
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/factory.py", line 188, in _make_candidate_from_link
+    self._editable_candidate_cache[link] = EditableCandidate(
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/candidates.py", line 313, in __init__
+    super().__init__(
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/candidates.py", line 158, in __init__
+    self.dist = self._prepare()
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/candidates.py", line 227, in _prepare
+    dist = self._prepare_distribution()
+  File "/usr/lib/python3/dist-packages/pip/_internal/resolution/resolvelib/candidates.py", line 323, in _prepare_distribution
+    return self._factory.preparer.prepare_editable_requirement(self._ireq)
+  File "/usr/lib/python3/dist-packages/pip/_internal/operations/prepare.py", line 609, in prepare_editable_requirement
+    dist = _get_prepared_distribution(
+  File "/usr/lib/python3/dist-packages/pip/_internal/operations/prepare.py", line 58, in _get_prepared_distribution
+    abstract_dist.prepare_distribution_metadata(finder, build_isolation)
+  File "/usr/lib/python3/dist-packages/pip/_internal/distributions/sdist.py", line 35, in prepare_distribution_metadata
+    self._prepare_build_backend(finder)
+  File "/usr/lib/python3/dist-packages/pip/_internal/distributions/sdist.py", line 56, in _prepare_build_backend
+    self.req.build_env.install_requirements(
+  File "/usr/lib/python3/dist-packages/pip/_internal/build_env.py", line 202, in install_requirements
+    self._install_requirements(
+  File "/usr/lib/python3/dist-packages/pip/_internal/build_env.py", line 260, in _install_requirements
+    call_subprocess(
+  File "/usr/lib/python3/dist-packages/pip/_internal/utils/subprocess.py", line 166, in call_subprocess
+    line: str = proc.stdout.readline()
+KeyboardInterrupt
+Removed file:///users/Haocheng/mylcs/libCacheSim-python from build tracker '/tmp/pip-req-tracker-g5ne9qc8'
+Removed build tracker: '/tmp/pip-req-tracker-g5ne9qc8'
+~/mylcs

libCacheSim-python/.gitignore

@@ -0,0 +1,59 @@
+# Automatically generated by `hgimportsvn`
+.svn
+.hgsvn
+
+# Ignore local virtualenvs
+lib/
+bin/
+include/
+.Python/
+
+# These lines are suggested according to the svn:ignore property
+# Feel free to enable them by uncommenting them
+*.pyc
+*.pyo
+*.swp
+*.class
+*.orig
+*~
+.hypothesis/
+
+# autogenerated
+src/_pytest/_version.py
+# setuptools
+.eggs/
+
+doc/*/_build
+doc/*/.doctrees
+build/
+dist/
+*.egg-info
+htmlcov/
+issue/
+env/
+.env/
+.venv/
+/pythonenv*/
+3rdparty/
+.tox
+.cache
+.pytest_cache
+.mypy_cache
+.coverage
+.coverage.*
+coverage.xml
+.ropeproject
+.idea
+.hypothesis
+.pydevproject
+.project
+.settings
+.vscode
+__pycache__/
+.python-version
+
+# generated by pip
+pip-wheel-metadata/
+
+# pytest debug logs generated via --debug
+pytestdebug.log

libCacheSim-python/CMakeLists.txt

@@ -0,0 +1,98 @@
+cmake_minimum_required(VERSION 3.15...3.27)
+
+# Include exported variables from cache
+set(PARENT_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../build")
+set(EXPORT_FILE "${PARENT_BUILD_DIR}/export_vars.cmake")
+
+if(EXISTS "${EXPORT_FILE}")
+    include("${EXPORT_FILE}")
+    message(STATUS "Loaded variables from export_vars.cmake")
+else()
+    message(FATAL_ERROR "export_vars.cmake not found at ${EXPORT_FILE}. Please build the main project first (e.g. cd .. && cmake -G Ninja -B build)")
+endif()
+
+# Force enable -fPIC
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+project(libCacheSim-python VERSION "${LIBCACHESIM_VERSION}")
+
+if(LOG_LEVEL_LOWER STREQUAL "default")
+    if(CMAKE_BUILD_TYPE_LOWER MATCHES "debug")
+        add_compile_definitions(LOGLEVEL=6)
+    else()
+        add_compile_definitions(LOGLEVEL=7)
+    endif()
+elseif(LOG_LEVEL_LOWER STREQUAL "verbose")
+    add_compile_definitions(LOGLEVEL=5)
+elseif(LOG_LEVEL_LOWER STREQUAL "debug")
+    add_compile_definitions(LOGLEVEL=6)
+elseif(LOG_LEVEL_LOWER STREQUAL "info")
+    add_compile_definitions(LOGLEVEL=7)
+elseif(LOG_LEVEL_LOWER STREQUAL "warn")
+    add_compile_definitions(LOGLEVEL=8)
+elseif(LOG_LEVEL_LOWER STREQUAL "error")
+    add_compile_definitions(LOGLEVEL=9)
+else()
+    add_compile_definitions(LOGLEVEL=7)
+endif()
+
+# Find python and pybind11
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(pybind11 CONFIG REQUIRED)
+
+# Include directories for dependencies
+include_directories(${GLib_INCLUDE_DIRS})
+include_directories(${GLib_CONFIG_INCLUDE_DIR})
+include_directories(${XGBOOST_INCLUDE_DIR})
+include_directories(${LIGHTGBM_PATH})
+include_directories(${ZSTD_INCLUDE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/bin)
+
+# Find the main libCacheSim library
+set(MAIN_PROJECT_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../build")
+set(MAIN_PROJECT_LIB_PATH "${MAIN_PROJECT_BUILD_DIR}/liblibCacheSim.a")
+
+if(EXISTS "${MAIN_PROJECT_LIB_PATH}")
+    message(STATUS "Found pre-built libCacheSim library at ${MAIN_PROJECT_LIB_PATH}")
+
+    # Import the main library as an imported target
+    add_library(libCacheSim_main STATIC IMPORTED)
+    set_target_properties(libCacheSim_main PROPERTIES
+        IMPORTED_LOCATION "${MAIN_PROJECT_LIB_PATH}"
+        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/include;${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/utils/include;${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim"
+    )
+
+    # Link dependencies that the main library needs
+    target_link_libraries(libCacheSim_main INTERFACE ${dependency_libs})
+    set(LIBCACHESIM_TARGET libCacheSim_main)
+
+else()
+    message(FATAL_ERROR "Pre-built libCacheSim library not found. Please build the main project first: cd .. && cmake -G Ninja -B build && ninja -C build")
+endif()
+
+python_add_library(_libcachesim MODULE
+    src/pylibcachesim.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/bin/cli_reader_utils.c
+    WITH_SOABI
+)
+
+set_target_properties(_libcachesim PROPERTIES
+    POSITION_INDEPENDENT_CODE ON
+    INSTALL_RPATH_USE_LINK_PATH TRUE
+    BUILD_WITH_INSTALL_RPATH TRUE
+    INSTALL_RPATH "$ORIGIN"
+)
+
+target_compile_definitions(_libcachesim PRIVATE VERSION_INFO=${PROJECT_VERSION})
+
+target_link_libraries(_libcachesim PRIVATE
+    ${LIBCACHESIM_TARGET}
+    pybind11::headers
+    pybind11::module
+    -Wl,--no-as-needed -ldl
+)
+
+# install to wheel directory
+install(TARGETS _libcachesim LIBRARY DESTINATION libcachesim)

libCacheSim-python/README.md

@@ -0,0 +1,44 @@
+# libCacheSim Python Binding
+
+Python bindings for libCacheSim, a high-performance cache simulator.
+
+## Installation
+
+```bash
+pip install .
+```
+
+## Development
+
+```bash
+pip install -e .
+```
+
+Test
+
+```
+python -m pytest .
+```
+
+## Usage
+
+```python
+import libcachesim as cachesim
+
+# Create a cache with FIFO eviction policy
+cache = cachesim.FIFO(cache_size=1024*1024)
+
+# Create a request
+req = cachesim.Request()
+req.obj_id = 1
+req.obj_size = 100
+
+# Check if object is in cache
+hit = cache.get(req)
+print(f"Cache hit: {hit}")
+```
+
+## Features
+
+- [x] Support for multiple eviction policies (FIFO, LRU, ARC, Clock, etc.)
+- [ ] trace analysis tools

libCacheSim-python/export/CMakeLists.txt

@@ -0,0 +1,28 @@
+# Helper functions are removed since we don't export source files anymore
+
+set(EXPORT_FILE "${CMAKE_BINARY_DIR}/export_vars.cmake")
+file(WRITE "${EXPORT_FILE}" "")
+
+# ==============================================================================
+# Export project metadata
+# ==============================================================================
+file(APPEND "${EXPORT_FILE}" "set(LIBCACHESIM_VERSION \"${${PROJECT_NAME}_VERSION}\")\n")
+
+# ==============================================================================
+# Export essential include directory variables
+# ==============================================================================
+foreach(var IN ITEMS GLib_INCLUDE_DIRS GLib_CONFIG_INCLUDE_DIR XGBOOST_INCLUDE_DIR LIGHTGBM_PATH ZSTD_INCLUDE_DIR)
+    file(APPEND "${EXPORT_FILE}" "set(${var} \"${${var}}\")\n")
+endforeach()
+
+# ==============================================================================
+# Export dependency library variables
+# ==============================================================================
+file(APPEND "${EXPORT_FILE}" "set(dependency_libs \"${dependency_libs}\")\n")
+
+# ==============================================================================
+# Export essential build option variables
+# ==============================================================================
+file(APPEND "${EXPORT_FILE}" "set(LOG_LEVEL_LOWER \"${LOG_LEVEL_LOWER}\")\n")
+
+message(STATUS "Exported essential variables to ${EXPORT_FILE}")