From be1209b664876cedcf5c1c1938fe1685ffe2f945 Mon Sep 17 00:00:00 2001 From: haochengxia Date: Wed, 25 Jun 2025 17:34:11 -0400 Subject: [PATCH 1/3] Use Ninja for faster build --- .github/workflows/build.yml | 15 ++-- .github/workflows/code-quality.yml | 18 ++--- .github/workflows/npm-release.yml | 56 +++++++-------- README.md | 79 +++++++++++----------- example/cacheCluster/README.md | 6 +- example/cacheHierarchy/README.md | 11 ++- example/cacheSimulator/README.md | 18 +++-- example/cacheSimulatorConcurrent/README.md | 18 +++-- example/plugin_v2/README.md | 4 +- scripts/debug.sh | 4 +- scripts/install_dependency.sh | 26 +++---- scripts/install_dev_dependency.sh | 6 +- scripts/install_libcachesim.sh | 4 +- scripts/setup_hooks.sh | 52 +++++++------- 14 files changed, 154 insertions(+), 163 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 860ffa74..a97903b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,9 +18,9 @@ jobs: - name: Prepare run: bash scripts/install_dependency.sh - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: ninja -C ${{github.workspace}}/build - name: Test working-directory: ${{github.workspace}}/build run: | @@ -34,12 +34,12 @@ jobs: run: bash scripts/install_dependency.sh - name: Configure CMake with LSan run: | - cmake -B ${{github.workspace}}/build \ + cmake -G Ninja -B ${{github.workspace}}/build \ -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ -DCMAKE_C_FLAGS="-fsanitize=leak" \ -DCMAKE_CXX_FLAGS="-fsanitize=leak" - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: ninja -C ${{github.workspace}}/build - name: Test with LSan working-directory: ${{github.workspace}}/build run: | @@ -51,12 +51,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on + run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on - name: Build - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + run: ninja -C ${{github.workspace}}/build - name: Test working-directory: ${{github.workspace}}/build run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel - - - diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 357f4830..22a94e21 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -14,7 +14,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y clang-format clang-tidy cmake g++ make + sudo apt-get install -y clang-format clang-tidy cmake g++ ninja-build bash scripts/install_dependency.sh - name: Get changed C/C++ files @@ -36,7 +36,7 @@ jobs: run: | echo "Running clang-format checks..." FORMAT_ERRORS=0 - + for file in ${{ steps.changed-files.outputs.all_changed_files }}; do echo "Checking formatting for $file" if ! clang-format --dry-run --Werror "$file" >> logs/format.log 2>&1; then @@ -44,7 +44,7 @@ jobs: FORMAT_ERRORS=$((FORMAT_ERRORS + 1)) fi done - + if [ $FORMAT_ERRORS -gt 0 ]; then echo "Found formatting issues in $FORMAT_ERRORS files." echo "You can fix these issues with: clang-format -i " @@ -57,7 +57,7 @@ jobs: - name: Configure CMake if: steps.changed-files.outputs.any_changed == 'true' run: | - cmake -B build -DCMAKE_BUILD_TYPE=Debug \ + cmake -G Ninja -B build -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wpedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -Wnested-externs -Wmissing-include-dirs" \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-pedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wmissing-include-dirs" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON @@ -68,11 +68,11 @@ jobs: # run: | # echo "Running clang-tidy checks..." # FILES_WITH_ISSUES=0 - + # for file in ${{ steps.changed-files.outputs.all_changed_files }}; do # echo "Checking $file with clang-tidy" # LOG_FILE="logs/tidy_$(basename "$file").log" - + # # Run clang-tidy with selected checks (same as pre-commit hook) # if ! clang-tidy -p=build \ # -checks='-*,bugprone-*,cert-*,clang-analyzer-*,cppcoreguidelines-*,performance-*,portability-*,readability-*,-readability-magic-numbers,-readability-braces-around-statements,-cppcoreguidelines-avoid-magic-numbers,-readability-identifier-length,-clang-diagnostic-unused-command-line-argument' \ @@ -82,7 +82,7 @@ jobs: # FILES_WITH_ISSUES=$((FILES_WITH_ISSUES + 1)) # fi # done - + # if [ $FILES_WITH_ISSUES -gt 0 ]; then # echo "clang-tidy found issues in $FILES_WITH_ISSUES files." # exit 1 @@ -94,10 +94,10 @@ jobs: if: steps.changed-files.outputs.any_changed == 'true' run: | echo "Checking for compilation warnings..." - if cmake --build build -j$(nproc) > logs/compile.log 2>&1; then + if ninja -C build > logs/compile.log 2>&1; then echo "Compilation successful!" else echo "Compilation failed. Please fix the warnings." cat logs/compile.log exit 1 - fi + fi diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml index ba293ee4..32b6f0b8 100644 --- a/.github/workflows/npm-release.yml +++ b/.github/workflows/npm-release.yml @@ -19,16 +19,16 @@ jobs: outputs: release_created: ${{ steps.release.outputs.release_created }} version: ${{ steps.package.outputs.version }} - + steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Synchronize Node.js binding version run: | echo "Synchronizing Node.js binding version with main project..." python3 scripts/sync_node_version.py - + - name: Get package version id: package working-directory: libCacheSim-node @@ -36,7 +36,7 @@ jobs: VERSION=$(node -p "require('./package.json').version") echo "version=$VERSION" >> $GITHUB_OUTPUT echo "Version: $VERSION" - + - name: Check if release exists id: check_release run: | @@ -50,7 +50,7 @@ jobs: fi env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + - name: Create GitHub Release id: release if: steps.check_release.outputs.exists == 'false' @@ -62,15 +62,15 @@ jobs: release_name: Release v${{ steps.package.outputs.version }} body: | Release v${{ steps.package.outputs.version }} - + ## Installation ```bash npm install libcachesim-node ``` - + ## Supported Platforms - Linux x64 - + Pre-compiled binaries are automatically downloaded during installation. draft: false prerelease: false @@ -79,48 +79,48 @@ jobs: if: github.event_name == 'release' needs: create-release runs-on: ubuntu-latest - + steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Synchronize Node.js binding version run: | echo "Synchronizing Node.js binding version with main project..." python3 scripts/sync_node_version.py - + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '18' registry-url: 'https://registry.npmjs.org' - + - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y build-essential cmake libglib2.0-dev libzstd-dev - + sudo apt-get install -y build-essential cmake ninja-build libglib2.0-dev libzstd-dev + - name: Build libCacheSim run: | echo "Building libCacheSim for Linux x64..." mkdir -p _build cd _build - cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} .. -DCMAKE_VERBOSE_MAKEFILE=ON + cmake -G Ninja -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} .. -DCMAKE_VERBOSE_MAKEFILE=ON echo "Starting build..." - make -j$(nproc) VERBOSE=1 - + ninja -v + - name: Prepare vendored library run: | mkdir -p libCacheSim-node/vendor/include cp _build/liblibCacheSim.a libCacheSim-node/vendor/ cp -r libCacheSim/include/* libCacheSim-node/vendor/include/ - + - name: Install Node.js dependencies working-directory: libCacheSim-node run: npm install - + - name: Build and upload prebuilt binary working-directory: libCacheSim-node run: | @@ -132,7 +132,7 @@ jobs: CFLAGS=-fPIC CXXFLAGS=-fPIC npx prebuild --upload-all --verbose env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + - name: Test binary working-directory: libCacheSim-node run: | @@ -145,33 +145,33 @@ jobs: if: github.event_name == 'release' needs: build-and-publish runs-on: ubuntu-latest - + steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Synchronize Node.js binding version run: | echo "Synchronizing Node.js binding version with main project..." python3 scripts/sync_node_version.py - + - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '18' registry-url: 'https://registry.npmjs.org' - + - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y build-essential cmake libglib2.0-dev libzstd-dev + sudo apt-get install -y build-essential cmake ninja-build libglib2.0-dev libzstd-dev - name: Build libCacheSim run: | mkdir -p _build cd _build - cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} .. - make -j$(nproc) + cmake -G Ninja -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} .. + ninja - name: Prepare vendored library run: | @@ -184,4 +184,4 @@ jobs: working-directory: libCacheSim-node run: npm publish --access public env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/README.md b/README.md index c1fb6c42..7d89f0db 100644 --- a/README.md +++ b/README.md @@ -41,23 +41,23 @@ * **2024 Oct**: **S3-FIFO** gets an upgrade! Please try out the new version (the old is now renamed to S3-FIFOv0). * **2023 June**: **QDLP** is available now, see [our paper](https://dl.acm.org/doi/10.1145/3593856.3595887) for details. * **2023 Oct**: **[S3-FIFO](https://dl.acm.org/doi/10.1145/3600006.3613147)** and **SIEVE(https://sievecache.com)** are available! These are very simple algorithms that are very effective in reducing cache misses. Try them out in libCacheSim and your production! -* **2024 Jan**: We compiled a list of open-source cache datasets at the bottom of this page +* **2024 Jan**: We compiled a list of open-source cache datasets at the bottom of this page --- ## What is libCacheSim -* a high-performance **cache simulator** for running cache simulations. +* a high-performance **cache simulator** for running cache simulations. * a high-performance and versatile trace analyzer for **analyzing different cache traces**. -* a high-performance **library** for building cache simulators. +* a high-performance **library** for building cache simulators. --- -## libCacheSim features -* **High performance** - over 20M requests/sec for a realistic trace replay. -* **High memory efficiency** - predictable and small memory footprint. -* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, prefetching algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). -* Parallelism out-of-the-box - uses the many CPU cores to speed up trace analysis and cache simulations. +## libCacheSim features +* **High performance** - over 20M requests/sec for a realistic trace replay. +* **High memory efficiency** - predictable and small memory footprint. +* **State-of-the-art algorithms** - eviction algorithms, admission algorithms, prefetching algorithms, sampling techniques, approximate miss ratio computation, see [here](/doc/quickstart_cachesim.md). +* Parallelism out-of-the-box - uses the many CPU cores to speed up trace analysis and cache simulations. * **The ONLY feature-rich trace analyzer** - all types of trace analysis you need, see [here](/doc/quickstart_traceAnalyzer.md). * **Simple API** - easy to build cache clusters, multi-layer caching, etc.; see [here](/doc/API.md). * **Extensible** - easy to support new trace types or eviction algorithms; see [here](/doc/advanced_lib_extend.md). @@ -100,18 +100,18 @@ cachesim supports the following algorithms: ## Build and Install libCacheSim ### One-line install -We provide some scripts for quick installation of libCacheSim. +We provide some scripts for quick installation of libCacheSim. ```bash cd scripts && bash install_dependency.sh && bash install_libcachesim.sh ``` -If this does not work, please +If this does not work, please 1. let us know what system you are using and what error you get 2. read the following sections for self-installation. ### Install dependency libCacheSim uses [cmake](https://cmake.org/) build system and has a few dependencies: [glib](https://developer.gnome.org/glib/), [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). -Please see [install.md](/doc/install.md) for instructions on how to install the dependencies. +Please see [install.md](/doc/install.md) for instructions on how to install the dependencies. @@ -121,8 +121,8 @@ cmake recommends **out-of-source build**, so we do it in a new directory: git clone https://github.com/1a1a11a/libCacheSim pushd libCacheSim mkdir _build && cd _build -cmake .. && make -j -[sudo] make install +cmake -G Ninja .. && ninja +[sudo] ninja install popd ``` @@ -151,7 +151,7 @@ The pre-commit hook: ## Usage ### cachesim (a high-performance cache simulator) -After building and installing libCacheSim, `cachesim` should be in the `_build/bin/` directory. +After building and installing libCacheSim, `cachesim` should be in the `_build/bin/` directory. #### basic usage ``` @@ -162,11 +162,11 @@ use `./bin/cachesim --help` to get more information. #### Run a single cache simulation -Run the example traces using the LRU eviction algorithm and a 1 GB cache size. +Run the example traces using the LRU eviction algorithm and a 1 GB cache size. ```bash # Note that no space between the cache size and the unit, and the unit is not case-sensitive -./bin/cachesim ../data/trace.vscsi vscsi lru 1gb +./bin/cachesim ../data/trace.vscsi vscsi lru 1gb ``` @@ -186,9 +186,9 @@ Run the example traces using the LRU eviction algorithm and a 1 GB cache size. # use a csv trace with more options ./bin/cachesim ../data/trace.csv csv lru 1gb -t "time-col=2, obj-id-col=5, obj-size-col=4, delimiter=,, has-header=true" -``` +``` -See [quick start cachesim](/doc/quickstart_cachesim.md) for more usages. +See [quick start cachesim](/doc/quickstart_cachesim.md) for more usages. #### Debug cachesim @@ -218,16 +218,16 @@ python3 plot_mrc_time.py --tracepath ../data/twitter_cluster52.csv --trace-forma python3 plot_appr_mrc.py SHARDS ../data/twitter_cluster52.vscsi vscsi 0.01 # plot miss ratio over size using Miniature Simulations -python3 plot_appr_mrc.py MINI ../data/twitter_cluster52.vscsi vscsi s3fifo "0.0001,0.0002,0.0004,0.0008,0.001,0.002,0.004,0.008,0.01,0.02,0.04,0.08,0.1,0.2,0.4,0.8" 0.001,0.01,0.1,1 --extra_args "--ignore-obj-size 1" +python3 plot_appr_mrc.py MINI ../data/twitter_cluster52.vscsi vscsi s3fifo "0.0001,0.0002,0.0004,0.0008,0.001,0.002,0.004,0.008,0.01,0.02,0.04,0.08,0.1,0.2,0.4,0.8" 0.001,0.01,0.1,1 --extra_args "--ignore-obj-size 1" ``` --- ### Trace analysis -libCacheSim also has a trace analyzer that provides a lot of useful information about the trace. -And it is very fast, designed to work with billions of requests. -It also comes with a set of scripts to help you analyze the trace. +libCacheSim also has a trace analyzer that provides a lot of useful information about the trace. +And it is very fast, designed to work with billions of requests. +It also comes with a set of scripts to help you analyze the trace. See [trace analysis](/doc/quickstart_traceAnalyzer.md) for more details. --- @@ -242,12 +242,12 @@ Constructing fine-grained miss ratio curves for large-scale workloads is very de --- -### Using libCacheSim as a library -libCacheSim can be used as a library for building cache simulators. +### Using libCacheSim as a library +libCacheSim can be used as a library for building cache simulators. For example, you can build a cache cluster with consistent hashing or a multi-layer cache simulator. -Here is a simplified example showing the basic APIs. -```c +Here is a simplified example showing the basic APIs. +```c #include /* open trace, see quickstart_lib.md for opening csv and binary trace */ @@ -257,8 +257,8 @@ reader_t *reader = open_trace("../data/trace.vscsi", VSCSI_TRACE, NULL); request_t *req = new_request(); /* create a LRU cache */ -common_cache_params_t cc_params = {.cache_size=1024*1024U}; -cache_t *cache = LRU_init(cc_params, NULL); +common_cache_params_t cc_params = {.cache_size=1024*1024U}; +cache_t *cache = LRU_init(cc_params, NULL); /* counters */ uint64_t n_req = 0, n_miss = 0; @@ -288,26 +288,26 @@ To run the executable, ./test.out ``` -See [here](/doc/advanced_lib.md) for more details, and see [example folder](/example) for examples on how to use libCacheSim, such as building a cache cluster with consistent hashing, multi-layer cache simulators. +See [here](/doc/advanced_lib.md) for more details, and see [example folder](/example) for examples on how to use libCacheSim, such as building a cache cluster with consistent hashing, multi-layer cache simulators. --- ### Extending libCacheSim (new algorithms and trace types) -libCacheSim supports *txt*, *csv*, and *binary* traces. We prefer binary traces because they allow libCacheSim to run faster, and the traces are more compact. +libCacheSim supports *txt*, *csv*, and *binary* traces. We prefer binary traces because they allow libCacheSim to run faster, and the traces are more compact. We also support zstd compressed binary traces without decompression. This allows you to store the traces with less space. -If you need to add a new trace type or a new algorithm, please see [here](/doc/advanced_lib_extend.md) for details. +If you need to add a new trace type or a new algorithm, please see [here](/doc/advanced_lib_extend.md) for details. -We encourage the users to check [deepWiki](https://deepwiki.com/1a1a11a/libCacheSim) for a more detailed documentation. +We encourage the users to check [deepWiki](https://deepwiki.com/1a1a11a/libCacheSim) for a more detailed documentation. --- ## Open source cache traces -In the [repo](/data/), there are sample traces in different formats (`csv`, `txt`, `vscsi`, and `oracleGeneral`). Note that the sampled traces are **very small** and __should not be used for evaluating different algorithms' miss ratios__. The full traces can be found either with the original release or the processed `oracleGeneral` format. +In the [repo](/data/), there are sample traces in different formats (`csv`, `txt`, `vscsi`, and `oracleGeneral`). Note that the sampled traces are **very small** and __should not be used for evaluating different algorithms' miss ratios__. The full traces can be found either with the original release or the processed `oracleGeneral` format. Note that the oracleGeneral traces are compressed with [zstd](https://github.com/facebook/zstd) and have the following format: @@ -319,7 +319,7 @@ struct { int64_t next_access_vtime; // -1 if no next access } ``` -The compressed traces can be used with libCacheSim without decompression. And libCacheSim provides a `tracePrint` tool to print the trace in a human-readable format. +The compressed traces can be used with libCacheSim without decompression. And libCacheSim provides a `tracePrint` tool to print the trace in a human-readable format. | Dataset | Year | Type | Original release | OracleGeneral format | @@ -332,16 +332,16 @@ The compressed traces can be used with libCacheSim without decompression. And li | MetaKV | 2022 | key-value | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaKV/) | | MetaCDN | 2023 | object | [link](https://cachelib.org/docs/Cache_Library_User_Guides/Cachebench_FB_HW_eval/#list-of-traces) | [link](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/metaCDN/) | -Among the large number of traces, I recommend using the newer ones from Twitter (cluster52), Wiki, and Meta. +Among the large number of traces, I recommend using the newer ones from Twitter (cluster52), Wiki, and Meta. --- -## Contributions +## Contributions We gladly welcome pull requests. -Before making any large changes, we recommend opening an issue and discussing your proposed changes. -If the changes are minor, then feel free to make them without discussion. -This project adheres to Google's coding style. By participating, you are expected to uphold this code. +Before making any large changes, we recommend opening an issue and discussing your proposed changes. +If the changes are minor, then feel free to make them without discussion. +This project adheres to Google's coding style. By participating, you are expected to uphold this code. --- @@ -394,4 +394,3 @@ See [LICENSE](LICENSE) for details. ## Related * [PyMimircache](https://github.com/1a1a11a/PyMimircache): a python based cache trace analysis platform, now deprecated --- - diff --git a/example/cacheCluster/README.md b/example/cacheCluster/README.md index 966bb1ae..0b15d00f 100644 --- a/example/cacheCluster/README.md +++ b/example/cacheCluster/README.md @@ -1,5 +1,5 @@ # a cache cluster example -This illustrate how to build a cache cluster using consistent hashing. +This illustrate how to build a cache cluster using consistent hashing. ## Build Please install libCacheSim first. @@ -7,8 +7,8 @@ Then you can run the following in this directory ```bash mkdir _build; cd _build; -cmake ..; -make -j; +cmake -G Ninja ..; +ninja; ``` diff --git a/example/cacheHierarchy/README.md b/example/cacheHierarchy/README.md index 240f2f88..59f543a3 100644 --- a/example/cacheHierarchy/README.md +++ b/example/cacheHierarchy/README.md @@ -1,6 +1,6 @@ # a cache hierarchy example -This simulates several L1 caches (each with one trace) and one L2 cache by first generating the misses of the L1 caches and feed in the L2 cache. -It outputs the L2 miss ratio curve. +This simulates several L1 caches (each with one trace) and one L2 cache by first generating the misses of the L1 caches and feed in the L2 cache. +It outputs the L2 miss ratio curve. ## Dependency @@ -11,12 +11,11 @@ It outputs the L2 miss ratio curve. ```bash mkdir _build/; cd _build/; -cmake ../; -make -j; +cmake -G Ninja ../; +ninja; ``` -## Run +## Run ./layeredCache ../config.yaml - diff --git a/example/cacheSimulator/README.md b/example/cacheSimulator/README.md index 356f3b65..d5f04815 100644 --- a/example/cacheSimulator/README.md +++ b/example/cacheSimulator/README.md @@ -1,5 +1,5 @@ -## Building a cache simulator -This tutorial shows how to build a cache simulator using libCacheSim. +## Building a cache simulator +This tutorial shows how to build a cache simulator using libCacheSim. There are three data structures required to build a cache simulator: 1. create a trace reader using `open_trace()` 2. create a cache using `LRU_init()` where LRU can be replaced by other cache replacement policies @@ -17,15 +17,15 @@ while (read_one_req(reader, req) == 0) { ``` See [main.c](main.c) for a complete example. -### Build +### Build ```bash -mkdir _build; +mkdir _build; cd _build; -cmake ..; -make; +cmake -G Ninja ..; +ninja; ``` -### Run +### Run ```bash ./cacheSimulator ``` @@ -35,6 +35,4 @@ make; Solution * run `sudo ldconfig` * check whether the library is under /usr/local/lib -* - - \ No newline at end of file +* diff --git a/example/cacheSimulatorConcurrent/README.md b/example/cacheSimulatorConcurrent/README.md index 4093815f..c4ee5582 100644 --- a/example/cacheSimulatorConcurrent/README.md +++ b/example/cacheSimulatorConcurrent/README.md @@ -1,24 +1,22 @@ -## Building a concurrent cache simulator +## Building a concurrent cache simulator -This tutorial shows how to build a concurrent cache simulator using libCacheSim. -The cache simulator can simulate multiple caches at the same time. +This tutorial shows how to build a concurrent cache simulator using libCacheSim. +The cache simulator can simulate multiple caches at the same time. This example illustrate the usage of `simulate_with_multi_caches()` and `simulate_at_multi_sizes()` APIs. The details of the APIs can be found at [simulator.h](../../libCacheSim/include/libCacheSim/simulator.h) This example also shows how to use a csv reader to read a csv trace. -### Build +### Build ```bash -mkdir _build; +mkdir _build; cd _build; -cmake ..; -make; +cmake -G Ninja ..; +ninja; ``` -### Run +### Run ```bash ./cacheSimulatorMultiSize ``` - - diff --git a/example/plugin_v2/README.md b/example/plugin_v2/README.md index f0a76f75..8c544837 100644 --- a/example/plugin_v2/README.md +++ b/example/plugin_v2/README.md @@ -15,8 +15,8 @@ To compile the plugin into a shared library: ```bash mkdir build cd build -cmake .. -make +cmake -G Ninja .. +ninja ``` This will create: diff --git a/scripts/debug.sh b/scripts/debug.sh index dc671ac4..60fc139c 100755 --- a/scripts/debug.sh +++ b/scripts/debug.sh @@ -61,12 +61,12 @@ cd _build_dbg # Configure and build with warning flags echo "Configuring and building project with strict warnings..." -cmake -DCMAKE_BUILD_TYPE=Debug \ +cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wpedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -Wnested-externs -Wmissing-include-dirs" \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -Wno-deprecated-copy -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-pedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wmissing-include-dirs" \ .. -make -j +ninja # Return to script directory # cd ${DIR} diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index 396a9aa3..fe7f1ceb 100755 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -102,7 +102,7 @@ fi setup_ubuntu() { log_info "Setting up Ubuntu dependencies..." sudo apt update - sudo apt install -yqq build-essential google-perftools xxhash + sudo apt install -yqq build-essential google-perftools xxhash ninja-build sudo apt install -yqq libglib2.0-dev libunwind-dev sudo apt install -yqq libgoogle-perftools-dev } @@ -124,7 +124,7 @@ setup_macOS() { log_error "Homebrew is not installed. Please install Homebrew first." exit 1 fi - brew install glib google-perftools argp-standalone xxhash llvm wget cmake zstd xgboost lightgbm + brew install glib google-perftools argp-standalone xxhash llvm wget cmake ninja zstd xgboost lightgbm } # Install CMake @@ -165,13 +165,13 @@ install_xgboost() { pushd xgboost >/dev/null mkdir -p build pushd build >/dev/null - cmake .. + cmake -G Ninja .. if [[ ${GITHUB_ACTIONS-} == "true" ]]; then - make + ninja else - make -j + ninja fi - sudo make install + sudo ninja install popd >/dev/null popd >/dev/null popd >/dev/null @@ -187,13 +187,13 @@ install_lightgbm() { pushd LightGBM >/dev/null mkdir -p build pushd build >/dev/null - cmake .. + cmake -G Ninja .. if [[ ${GITHUB_ACTIONS-} == "true" ]]; then - make + ninja else - make -j + ninja fi - sudo make install + sudo ninja install popd >/dev/null popd >/dev/null popd >/dev/null @@ -211,9 +211,9 @@ install_zstd() { pushd "zstd-${zstd_version}/build/cmake/" >/dev/null mkdir -p _build pushd _build >/dev/null - cmake .. - make -j - sudo make install + cmake -G Ninja .. + ninja + sudo ninja install popd >/dev/null popd >/dev/null popd >/dev/null diff --git a/scripts/install_dev_dependency.sh b/scripts/install_dev_dependency.sh index 0f76cff3..daebbeed 100755 --- a/scripts/install_dev_dependency.sh +++ b/scripts/install_dev_dependency.sh @@ -41,16 +41,16 @@ FILE_DIR=$(dirname "$0") main() { # Detect OS and setup basic dependencies if [[ -f /etc/os-release ]] && grep -qiE 'ubuntu|debian' /etc/os-release; then - sudo apt install -yqq gdb valgrind clang-tidy clang-format + sudo apt install -yqq gdb valgrind clang-tidy clang-format ninja-build # node js dependency for libCacheSim-js sudo apt install -yqq nodejs npm # trunk-ignore(shellcheck/SC2312) elif [[ $(uname -a) == *"Darwin"* ]]; then # note that clang-format version on macOS might be newer than the one in ubuntu, and it can cause different formatting results - brew install gdb clang-format flock nodejs npm + brew install gdb clang-format flock nodejs npm ninja elif grep -qi 'microsoft' /proc/version 2>/dev/null; then # WSL detection - sudo apt install -yqq gdb valgrind clang-tidy clang-format + sudo apt install -yqq gdb valgrind clang-tidy clang-format ninja-build else log_error "Unsupported operating system. Only Ubuntu, Debian, WSL, and macOS are supported. Some dependencies may not be installed." fi diff --git a/scripts/install_libcachesim.sh b/scripts/install_libcachesim.sh index a0920cdf..4c9ad537 100755 --- a/scripts/install_libcachesim.sh +++ b/scripts/install_libcachesim.sh @@ -7,6 +7,6 @@ DIR=$(dirname "${SOURCE}") cd "${DIR}"/../ mkdir -p _build cd _build -cmake .. -make -j +cmake -G Ninja .. +ninja cd "${DIR}" diff --git a/scripts/setup_hooks.sh b/scripts/setup_hooks.sh index 7ac53628..5e5c86b2 100755 --- a/scripts/setup_hooks.sh +++ b/scripts/setup_hooks.sh @@ -61,7 +61,7 @@ echo -e "${BLUE}Configuring temporary build for linting...${NC}" mkdir -p "$TEMP_BUILD_DIR" cd "$TEMP_BUILD_DIR" CMAKE_LOG="$LOG_DIR/cmake.log" -cmake -DCMAKE_BUILD_TYPE=Debug \ +cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wpedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -Wnested-externs -Wmissing-include-dirs" \ -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-pedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wmissing-include-dirs" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ @@ -93,21 +93,21 @@ show_elapsed() { if command -v clang-format >/dev/null 2>&1; then echo -e "${BLUE}Running clang-format checks in parallel (max $MAX_JOBS jobs)...${NC}" FORMAT_TIME=$(date +%s) - + # Create a temporary file to capture errors FORMAT_ERROR_LOG="$LOG_DIR/format_errors.log" touch "$FORMAT_ERROR_LOG" - + # Create a file to track error count from subshells FORMAT_ERROR_COUNT_FILE="$LOG_DIR/format_error_count.txt" echo "0" > "$FORMAT_ERROR_COUNT_FILE" - + cd "$GIT_ROOT" - + # Run clang-format on each file in parallel PIDS=() FILES_CHECKED=0 - + for FILE in $STAGED_FILES; do # If we've hit the max jobs limit, wait for one to finish while [ ${#PIDS[@]} -ge $MAX_JOBS ]; do @@ -120,21 +120,21 @@ if command -v clang-format >/dev/null 2>&1; then # Only wait a bit before checking again sleep 0.1 done - + # Track progress FILES_CHECKED=$((FILES_CHECKED + 1)) PROGRESS=$((FILES_CHECKED * 100 / NUM_FILES)) echo -e " [${PROGRESS}%] Checking formatting for ${BLUE}$FILE${NC}" - + # Create a log file for this specific file FORMAT_LOG="$LOG_DIR/format_$(basename "$FILE").log" - + # Run clang-format in background ( if ! clang-format --dry-run --Werror "$FILE" > "$FORMAT_LOG" 2>&1; then echo -e " ${YELLOW}Formatting issues in $FILE (see $FORMAT_LOG)${NC}" echo "$FILE" >> "$FORMAT_ERROR_LOG" - + # Update the error count in the shared file flock "$FORMAT_ERROR_COUNT_FILE" bash -c " COUNT=\$(cat \"$FORMAT_ERROR_COUNT_FILE\") @@ -142,11 +142,11 @@ if command -v clang-format >/dev/null 2>&1; then " fi ) & - + # Store the PID PIDS+=($!) done - + # Wait for all remaining jobs to finish with a progress indicator echo -e "${BLUE}Waiting for all clang-format jobs to complete...${NC}" while [ ${#PIDS[@]} -gt 0 ]; do @@ -159,10 +159,10 @@ if command -v clang-format >/dev/null 2>&1; then sleep 1 done echo -e "\n${BLUE}All clang-format jobs completed.${NC}" - + # Read the final error count FORMAT_ERRORS=$(cat "$FORMAT_ERROR_COUNT_FILE") - + # Report the results if [ -s "$FORMAT_ERROR_LOG" ]; then echo -e "${RED}Found formatting issues in $FORMAT_ERRORS files. Commit blocked.${NC}" @@ -185,19 +185,19 @@ fi # if command -v clang-tidy >/dev/null 2>&1; then # echo -e "${BLUE}Running clang-tidy checks in parallel (max $MAX_JOBS jobs)...${NC}" # TIDY_TIME=$(date +%s) - + # # Create a temporary file to capture errors # ERROR_LOG="$LOG_DIR/tidy_errors.log" # touch "$ERROR_LOG" - + # # Create a file to track error count from subshells # ERROR_COUNT_FILE="$LOG_DIR/error_count.txt" # echo "0" > "$ERROR_COUNT_FILE" - + # # Run clang-tidy on each file in parallel # PIDS=() # FILES_CHECKED=0 - + # for FILE in $STAGED_FILES; do # # If we've hit the max jobs limit, wait for one to finish # while [ ${#PIDS[@]} -ge $MAX_JOBS ]; do @@ -210,15 +210,15 @@ fi # # Only wait a bit before checking again # sleep 0.1 # done - + # # Track progress # FILES_CHECKED=$((FILES_CHECKED + 1)) # PROGRESS=$((FILES_CHECKED * 100 / NUM_FILES)) # echo -e " [${PROGRESS}%] Checking ${BLUE}$FILE${NC} with clang-tidy" - + # # Create a log file for this specific file # TIDY_LOG="$LOG_DIR/tidy_$(basename "$FILE").log" - + # # Run clang-tidy in background # ( # # Run clang-tidy with a selected set of checks @@ -228,7 +228,7 @@ fi # "$GIT_ROOT/$FILE" > "$TIDY_LOG" 2>&1; then # echo -e " ${YELLOW}clang-tidy found issues in $FILE (see $TIDY_LOG)${NC}" # echo "$FILE" >> "$ERROR_LOG" - + # # Update the error count in the shared file # flock "$ERROR_COUNT_FILE" bash -c " # COUNT=\$(cat \"$ERROR_COUNT_FILE\") @@ -236,11 +236,11 @@ fi # " # fi # ) & - + # # Store the PID # PIDS+=($!) # done - + # # Wait for all remaining jobs to finish with a progress indicator # echo -e "${BLUE}Waiting for all clang-tidy jobs to complete...${NC}" # while [ ${#PIDS[@]} -gt 0 ]; do @@ -253,10 +253,10 @@ fi # sleep 1 # done # echo -e "\n${BLUE}All clang-tidy jobs completed.${NC}" - + # # Read the final error count # FILES_WITH_ISSUES=$(cat "$ERROR_COUNT_FILE") - + # # Report the results # if [ -s "$ERROR_LOG" ]; then # echo -e "${RED}clang-tidy found issues in $FILES_WITH_ISSUES files. Commit blocked.${NC}" From 06f8b65955ff7f59e369580c636baf53681a2688 Mon Sep 17 00:00:00 2001 From: Percy Date: Thu, 26 Jun 2025 20:34:07 -0400 Subject: [PATCH 2/3] Build with multicore --- .github/workflows/build.yml | 22 ++++++++++++++++------ scripts/install_dependency.sh | 17 ++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a97903b6..6f010aff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,14 @@ jobs: - name: Configure CMake run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - name: Build - run: ninja -C ${{github.workspace}}/build + run: | + cores=$(sysctl -n hw.ncpu) + ninja -C ${{github.workspace}}/build -j$((cores + 1)) - name: Test working-directory: ${{github.workspace}}/build run: | - ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel + cores=$(sysctl -n hw.ncpu) + ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores ubuntu: runs-on: ubuntu-latest @@ -39,12 +42,15 @@ jobs: -DCMAKE_C_FLAGS="-fsanitize=leak" \ -DCMAKE_CXX_FLAGS="-fsanitize=leak" - name: Build - run: ninja -C ${{github.workspace}}/build + run: | + cores=$(nproc) + ninja -C ${{github.workspace}}/build -j$((cores + 1)) - name: Test with LSan working-directory: ${{github.workspace}}/build run: | export ASAN_OPTIONS="detect_leaks=1:halt_on_error=1:verbosity=1" - ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel + cores=$(nproc) + ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores selfhosted: runs-on: self-hosted @@ -53,7 +59,11 @@ jobs: - name: Configure CMake run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on - name: Build - run: ninja -C ${{github.workspace}}/build + run: | + cores=$(nproc) + ninja -C ${{github.workspace}}/build -j$((cores + 1)) - name: Test working-directory: ${{github.workspace}}/build - run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel + run: | + cores=$(nproc) + ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index fe7f1ceb..52e91a0c 100755 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -158,6 +158,7 @@ install_cmake() { # Install XGBoost install_xgboost() { log_info "Installing XGBoost..." + local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null if [[ ! -d "xgboost" ]]; then git clone --recursive https://github.com/dmlc/xgboost @@ -166,11 +167,7 @@ install_xgboost() { mkdir -p build pushd build >/dev/null cmake -G Ninja .. - if [[ ${GITHUB_ACTIONS-} == "true" ]]; then - ninja - else - ninja - fi + ninja -j$((cores + 1)) sudo ninja install popd >/dev/null popd >/dev/null @@ -180,6 +177,7 @@ install_xgboost() { # Install LightGBM install_lightgbm() { log_info "Installing LightGBM..." + local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null if [[ ! -d "LightGBM" ]]; then git clone --recursive https://github.com/microsoft/LightGBM @@ -188,11 +186,7 @@ install_lightgbm() { mkdir -p build pushd build >/dev/null cmake -G Ninja .. - if [[ ${GITHUB_ACTIONS-} == "true" ]]; then - ninja - else - ninja - fi + ninja -j$((cores + 1)) sudo ninja install popd >/dev/null popd >/dev/null @@ -202,6 +196,7 @@ install_lightgbm() { # Install Zstd install_zstd() { log_info "Installing Zstd..." + local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null local zstd_version="1.5.0" if [[ ! -f "zstd-${zstd_version}.tar.gz" ]]; then @@ -212,7 +207,7 @@ install_zstd() { mkdir -p _build pushd _build >/dev/null cmake -G Ninja .. - ninja + ninja -j$((cores + 1)) sudo ninja install popd >/dev/null popd >/dev/null From 7b3624fa68ba72c9061be9ef3f10d79233822e5e Mon Sep 17 00:00:00 2001 From: Percy Date: Thu, 26 Jun 2025 20:49:57 -0400 Subject: [PATCH 3/3] Clean up --- .github/workflows/build.yml | 23 ++++++---------------- README.md | 5 +++++ doc/install.md | 20 +++++++++---------- doc/memory_usage_profiling.md | 18 ++++++++--------- dockerfile | 10 ++++------ example/cacheCluster/README.md | 4 ++++ example/cacheHierarchy/README.md | 4 ++++ example/cacheSimulator/README.md | 4 ++++ example/cacheSimulatorConcurrent/README.md | 4 ++++ example/plugin_v2/README.md | 4 ++++ libCacheSim-node/README.md | 6 +++--- scripts/install_dependency.sh | 9 +++------ scripts/setup_hooks.sh | 2 +- 13 files changed, 61 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f010aff..df760719 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,14 +20,10 @@ jobs: - name: Configure CMake run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} - name: Build - run: | - cores=$(sysctl -n hw.ncpu) - ninja -C ${{github.workspace}}/build -j$((cores + 1)) + run: ninja -C ${{github.workspace}}/build - name: Test working-directory: ${{github.workspace}}/build - run: | - cores=$(sysctl -n hw.ncpu) - ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores + run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel ubuntu: runs-on: ubuntu-latest @@ -42,15 +38,12 @@ jobs: -DCMAKE_C_FLAGS="-fsanitize=leak" \ -DCMAKE_CXX_FLAGS="-fsanitize=leak" - name: Build - run: | - cores=$(nproc) - ninja -C ${{github.workspace}}/build -j$((cores + 1)) + run: ninja -C ${{github.workspace}}/build - name: Test with LSan working-directory: ${{github.workspace}}/build run: | export ASAN_OPTIONS="detect_leaks=1:halt_on_error=1:verbosity=1" - cores=$(nproc) - ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores + ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel selfhosted: runs-on: self-hosted @@ -59,11 +52,7 @@ jobs: - name: Configure CMake run: cmake -G Ninja -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DENABLE_GLCACHE=on -DENABLE_LRB=on - name: Build - run: | - cores=$(nproc) - ninja -C ${{github.workspace}}/build -j$((cores + 1)) + run: ninja -C ${{github.workspace}}/build - name: Test working-directory: ${{github.workspace}}/build - run: | - cores=$(nproc) - ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel $cores + run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --parallel diff --git a/README.md b/README.md index 7d89f0db..b026fce5 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,11 @@ Please see [install.md](/doc/install.md) for instructions on how to install the ### Build libCacheSim cmake recommends **out-of-source build**, so we do it in a new directory: ```bash +# Prerequisites: Install Ninja build system +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja +# CentOS/RHEL: sudo yum install ninja-build + git clone https://github.com/1a1a11a/libCacheSim pushd libCacheSim mkdir _build && cd _build diff --git a/doc/install.md b/doc/install.md index 4c49d08e..ec91f79b 100644 --- a/doc/install.md +++ b/doc/install.md @@ -1,16 +1,16 @@ ## Install dependency -libCacheSim uses [cmake](https://cmake.org/) build system and has a few dependencies: +libCacheSim uses [cmake](https://cmake.org/) build system with [Ninja](https://ninja-build.org/) generator and has a few dependencies: [glib](https://developer.gnome.org/glib/) [tcmalloc](https://github.com/google/tcmalloc), [zstd](https://github.com/facebook/zstd). ### Install dependency on Ubuntu -#### Install glib, tcmalloc and cmake +#### Install glib, tcmalloc, cmake and ninja ```bash -sudo apt install libglib2.0-dev libgoogle-perftools-dev cmake +sudo apt install libglib2.0-dev libgoogle-perftools-dev cmake ninja-build ``` #### Install zstd @@ -22,8 +22,8 @@ wget https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz tar xvf zstd-1.5.0.tar.gz pushd zstd-1.5.0/build/cmake/ mkdir _build && cd _build/ -cmake .. && make -j -sudo make install +cmake -G Ninja .. && ninja +sudo ninja install popd ``` @@ -33,8 +33,8 @@ popd git clone --recursive https://github.com/dmlc/xgboost pushd xgboost mkdir _build && cd _build -cmake .. && make -j -sudo make install +cmake -G Ninja .. && ninja +sudo ninja install popd ``` @@ -44,8 +44,8 @@ popd git clone --recursive https://github.com/microsoft/LightGBM pushd LightGBM mkdir _build && cd _build -cmake .. && make -j -sudo make install +cmake -G Ninja .. && ninja +sudo ninja install popd ``` @@ -57,7 +57,7 @@ using [homebrew](https://brew.sh/) as an example. While the first line is necess - /usr/local/include/glib-2.0/glib/gtypes.h:34:10: fatal error: 'glibconfig.h' file not found ```bash -brew install cmake glib google-perftools +brew install cmake ninja glib google-perftools brew install argp-standalone brew install pkg-config ``` diff --git a/doc/memory_usage_profiling.md b/doc/memory_usage_profiling.md index 1f91f53b..024e5c0e 100644 --- a/doc/memory_usage_profiling.md +++ b/doc/memory_usage_profiling.md @@ -1,7 +1,7 @@ # Profiling Memory Usage in CacheSim with Massif ## 1. Introduction -This guide explains how to analyze memory usage in the `cachesim` project using Valgrind's **massif** tool and identifies critical memory overhead areas as potential optimization targets. The workflow consists of the following steps: +This guide explains how to analyze memory usage in the `cachesim` project using Valgrind's **massif** tool and identifies critical memory overhead areas as potential optimization targets. The workflow consists of the following steps: - Heap memory profiling - Identifying memory hotspots @@ -15,8 +15,8 @@ This guide explains how to analyze memory usage in the `cachesim` project using ```sh # Compile with debug symbols mkdir _build && cd _build -cmake .. -DCMAKE_BUILD_TYPE=Debug -make -j +cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Debug +ninja ``` 2. Run `massif` for a specific algorithm (e.g., S3-FIFO): @@ -38,7 +38,7 @@ The generated report primarily includes a bar chart of memory usage (with instru ```sh MB -519.4^ : +519.4^ : |#:::::::::::::@::::::::::::::::@::@@::::::::::::::::::::::::@::@::@::@:: |#::: :: :: :::@: ::::: : ::: ::@::@ ::: :: : : ::::::: :::::@::@::@::@:: |#::: :: :: :::@: ::::: : ::: ::@::@ ::: :: : : ::::::: :::::@::@::@::@:: @@ -76,22 +76,22 @@ The generated report primarily includes a bar chart of memory usage (with instru | | | ->24.74% (134,217,728B) 0x40880A: create_cache (cache_init.h:132) | | | ->24.74% (134,217,728B) 0x40970B: parse_cmd (cli_parser.c:343) | | | ->24.74% (134,217,728B) 0x406FFD: main (main.c:18) -| | | +| | | | | ->24.74% (134,217,728B) 0x4238C0: S3FIFO_init (S3FIFO.c:118) | | | ->24.74% (134,217,728B) 0x40880A: create_cache (cache_init.h:132) | | | ->24.74% (134,217,728B) 0x40970B: parse_cmd (cli_parser.c:343) | | | ->24.74% (134,217,728B) 0x406FFD: main (main.c:18) -| | | +| | | | | ->24.74% (134,217,728B) 0x423921: S3FIFO_init (S3FIFO.c:125) | | ->24.74% (134,217,728B) 0x40880A: create_cache (cache_init.h:132) | | ->24.74% (134,217,728B) 0x40970B: parse_cmd (cli_parser.c:343) | | ->24.74% (134,217,728B) 0x406FFD: main (main.c:18) -| | +| | | ->24.74% (134,217,728B) 0x4236BB: S3FIFO_init (S3FIFO.c:81) | ->24.74% (134,217,728B) 0x40880A: create_cache (cache_init.h:132) | ->24.74% (134,217,728B) 0x40970B: parse_cmd (cli_parser.c:343) | ->24.74% (134,217,728B) 0x406FFD: main (main.c:18) -| +| ->00.84% (4,541,568B) in 1+ places, all below ms_print's threshold (01.00%) ``` @@ -105,7 +105,7 @@ The above report indicates that the memory usage of S3-FIFO remains nearly const ### 3.1. Memory Usage of Various Algorithms -We conducted tests using the `wiki_2019t` workload (available at [CMU's dataset repository](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/wiki/)) to evaluate the memory usage of various algorithms with a cache size of 1GB. +We conducted tests using the `wiki_2019t` workload (available at [CMU's dataset repository](https://ftp.pdl.cmu.edu/pub/datasets/twemcacheWorkload/cacheDatasets/wiki/)) to evaluate the memory usage of various algorithms with a cache size of 1GB. | Algorithm | Peak Memory Usage (MB) | Observations | diff --git a/dockerfile b/dockerfile index 9cb5f708..54cb4261 100644 --- a/dockerfile +++ b/dockerfile @@ -21,7 +21,7 @@ RUN bash ./scripts/install_libcachesim.sh WORKDIR /libCacheSim/_build/ -# RUN cmake -DSUPPORT_ZSTD_TRACE=on .. && make -j && sudo make install +# RUN cmake -G Ninja -DSUPPORT_ZSTD_TRACE=on .. && ninja && sudo ninja install WORKDIR /libCacheSim/_build/bin/ @@ -40,9 +40,9 @@ WORKDIR /libCacheSim/_build/bin/ # sudo docker run -v /local/data/path:/data -it 1a1a11a/libcachesim:latest bash # docker install instructions can be found at https://docs.docker.com/engine/install/ -# for ubuntu: -# -# sudo apt-get update && sudo apt-get install -yqq ca-certificates curl gnupg; +# for ubuntu: +# +# sudo apt-get update && sudo apt-get install -yqq ca-certificates curl gnupg; # sudo install -m 0755 -d /etc/apt/keyrings && \ # curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ # sudo chmod a+r /etc/apt/keyrings/docker.gpg @@ -51,5 +51,3 @@ WORKDIR /libCacheSim/_build/bin/ # "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ # sudo tee /etc/apt/sources.list.d/docker.list > /dev/null # sudo apt update && sudo apt-get install -yqq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - diff --git a/example/cacheCluster/README.md b/example/cacheCluster/README.md index 0b15d00f..829610ae 100644 --- a/example/cacheCluster/README.md +++ b/example/cacheCluster/README.md @@ -5,6 +5,10 @@ This illustrate how to build a cache cluster using consistent hashing. Please install libCacheSim first. Then you can run the following in this directory ```bash +# Prerequisites: Install Ninja build system if not already available +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja + mkdir _build; cd _build; cmake -G Ninja ..; diff --git a/example/cacheHierarchy/README.md b/example/cacheHierarchy/README.md index 59f543a3..5a208309 100644 --- a/example/cacheHierarchy/README.md +++ b/example/cacheHierarchy/README.md @@ -9,6 +9,10 @@ It outputs the L2 miss ratio curve. ## Build ```bash +# Prerequisites: Install Ninja build system if not already available +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja + mkdir _build/; cd _build/; cmake -G Ninja ../; diff --git a/example/cacheSimulator/README.md b/example/cacheSimulator/README.md index d5f04815..74e4d80f 100644 --- a/example/cacheSimulator/README.md +++ b/example/cacheSimulator/README.md @@ -19,6 +19,10 @@ See [main.c](main.c) for a complete example. ### Build ```bash +# Prerequisites: Install Ninja build system if not already available +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja + mkdir _build; cd _build; cmake -G Ninja ..; diff --git a/example/cacheSimulatorConcurrent/README.md b/example/cacheSimulatorConcurrent/README.md index c4ee5582..62d3812e 100644 --- a/example/cacheSimulatorConcurrent/README.md +++ b/example/cacheSimulatorConcurrent/README.md @@ -10,6 +10,10 @@ This example also shows how to use a csv reader to read a csv trace. ### Build ```bash +# Prerequisites: Install Ninja build system if not already available +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja + mkdir _build; cd _build; cmake -G Ninja ..; diff --git a/example/plugin_v2/README.md b/example/plugin_v2/README.md index 8c544837..ce892005 100644 --- a/example/plugin_v2/README.md +++ b/example/plugin_v2/README.md @@ -13,6 +13,10 @@ This example demonstrates how to create a plugin for libCacheSim using the v2 ho To compile the plugin into a shared library: ```bash +# Prerequisites: Install Ninja build system if not already available +# Ubuntu/Debian: sudo apt install ninja-build +# macOS: brew install ninja + mkdir build cd build cmake -G Ninja .. diff --git a/libCacheSim-node/README.md b/libCacheSim-node/README.md index f26d5ca5..8cebf1ee 100644 --- a/libCacheSim-node/README.md +++ b/libCacheSim-node/README.md @@ -189,9 +189,9 @@ If you see binary loading errors: If source compilation fails: -1. Install required system dependencies +1. Install required system dependencies (including ninja-build) 2. Ensure CMake 3.10+ is available -3. Check that libCacheSim builds successfully: `cd .. && mkdir _build && cd _build && cmake .. && make` +3. Check that libCacheSim builds successfully: `cd .. && mkdir _build && cd _build && cmake -G Ninja .. && ninja` ## Contributing @@ -217,4 +217,4 @@ If you use libCacheSim in your research, please cite: year={2023}, url={https://github.com/1a1a11a/libCacheSim} } -``` \ No newline at end of file +``` diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index 52e91a0c..5da833da 100755 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -158,7 +158,6 @@ install_cmake() { # Install XGBoost install_xgboost() { log_info "Installing XGBoost..." - local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null if [[ ! -d "xgboost" ]]; then git clone --recursive https://github.com/dmlc/xgboost @@ -167,7 +166,7 @@ install_xgboost() { mkdir -p build pushd build >/dev/null cmake -G Ninja .. - ninja -j$((cores + 1)) + ninja sudo ninja install popd >/dev/null popd >/dev/null @@ -177,7 +176,6 @@ install_xgboost() { # Install LightGBM install_lightgbm() { log_info "Installing LightGBM..." - local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null if [[ ! -d "LightGBM" ]]; then git clone --recursive https://github.com/microsoft/LightGBM @@ -186,7 +184,7 @@ install_lightgbm() { mkdir -p build pushd build >/dev/null cmake -G Ninja .. - ninja -j$((cores + 1)) + ninja sudo ninja install popd >/dev/null popd >/dev/null @@ -196,7 +194,6 @@ install_lightgbm() { # Install Zstd install_zstd() { log_info "Installing Zstd..." - local cores=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2) pushd /tmp/ >/dev/null local zstd_version="1.5.0" if [[ ! -f "zstd-${zstd_version}.tar.gz" ]]; then @@ -207,7 +204,7 @@ install_zstd() { mkdir -p _build pushd _build >/dev/null cmake -G Ninja .. - ninja -j$((cores + 1)) + ninja sudo ninja install popd >/dev/null popd >/dev/null diff --git a/scripts/setup_hooks.sh b/scripts/setup_hooks.sh index 5e5c86b2..3c106396 100755 --- a/scripts/setup_hooks.sh +++ b/scripts/setup_hooks.sh @@ -278,7 +278,7 @@ fi # cd "$TEMP_BUILD_DIR" # COMPILE_LOG="$LOG_DIR/compile.log" -# if make -j$MAX_JOBS > "$COMPILE_LOG" 2>&1; then +# if ninja > "$COMPILE_LOG" 2>&1; then # echo -e "${GREEN}Compilation successful!${NC}" # else # echo -e "${RED}Compilation failed. Please fix the warnings before committing.${NC}"