diff --git a/.gitignore b/.gitignore index 957c0f33..9913f147 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ example/cacheSimulatorC/cmake-build-debug *.log fig/ result/ +data_large/ # Chaos sftp-config.json # Clangd cache diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 23d2dd05..00000000 --- a/.travis.yml +++ /dev/null @@ -1,178 +0,0 @@ -language: c - -env: - global: - # Ubuntu version -# - LINUX_DIST=trusty - - MAKEFLAGS="-j 2" - - RUN_TESTS=true - - COVERAGE=false - -matrix: - include: - - os: linux -# dist: trusty - sudo: true - compiler: gcc - addons: - apt: - update: true - packages: - - gcc - - g++ - #- gcov-4.9 - - clang - - python-yaml - - lcov - - libglib2.0-dev - - libgoogle-perftools-dev - - pkg-config - - os: osx - osx_image: xcode11.4 - compiler: clang - homebrew: - update: true - packages: - - glib - - gcc - - pkg-config - - google-perftools - -script: - ############################################################################ - # Build main and tests - ############################################################################ - - mkdir -p _build - - cd _build - - cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON .. - - make -j -# - make project1_coverage - - ctest - - sudo make install - - bash ../test/test_lib.sh - -#after_success: -# - codecov -t c376990f-2ad6-4f7b-bbdb-5378a52e295c -X gcov - - - - - - -#language: python -# -## os: -## - linux -## - osx -# -##python: -## - "3.4" -## - "3.5" -## - "3.6" -## - "nightly" -# -#addons: -# apt: -## sources: -## - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu/ trusty main restricted universe multiverse' -## - sourceline: 'deb-src http://us.archive.ubuntu.com/ubuntu/ trusty main restricted universe multiverse' -# packages: -# libglib2.0-dev -# python3-pip -# python3-matplotlib -# python3-dev -# git -# gcc -# g++ -# -#before_install: -## - sudo apt-get -qq update -## - sudo apt-get install -y libglib2.0-dev python3-pip python3-matplotlib python3-dev git gcc g++ -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install glib python3 openssl readline ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew outdated pyenv || brew upgrade pyenv ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install pyenv-virtualenv ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/Users/travis/.pyenv/shims:${PATH}" ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pyenv-virtualenv venv; source venv/bin/activate ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python --version ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m pip install -U pip ; fi -# - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python -m easy_install -U setuptools ; fi -# -# -## command to install dependencies -#install: -# - pip3 install --upgrade pip setuptools wheel -# - pip3 install -r requirements.txt -# - python3 setup.py build_ext -i -# - which python3 -# - python3 --version -# -#matrix: -# include: -# - os: linux -# python: 3.4 -# - os: linux -# python: 3.5 -# - os: linux -# python: 3.6 -# allow_failures: -# - os: linux -# python: "nightly" -## - os: osx -## language: generic -# -# -# -## command to run tests -#script: nosetests - - - - - -# command to install dependencies -#install: "pip3 install -r requirements.txt; python3 setup.py build_ext -i" -# command to run tests -#script: nosetests - - - - - - - - - - - - - -## using anchor to import sources into linux builds -#addons: -# apt: &apt -# sources: -# - ubuntu-toolchain-r-test - -# important for allowed-to-fail matching -# see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail - -# travis currently does not support directly setting gcc/clang with versions -# (e.g. gcc-4.8) as value for the compiler key. So we will have to manually -# request these packages and use environment varibles to create the matrix. -#jobs: -# fast_finish: true -# include: -# - name: "gcc on Linux" -# compiler: gcc -# -# - os: osx -# osx_image: xcode11.4 -# compiler: clang -# -# allow_failures: -# - os: osx -# osx_image: xcode11.4 -# compiler: clang -# -#script: -# - ./ci/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index a89d8ead..925fa40e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,10 +16,7 @@ set(CMAKE_CXX_EXTENSIONS Off) # export compile commands, useful for IDEs set(EXPORT_COMPILE_COMMANDS ON) -# ####################################### -# define options # -# options are cached variables https://stackoverflow.com/questions/35744647/disabling-cmake-option-has-no-effect -# ####################################### +# Define build options # echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled option(USE_HUGEPAGE "use transparent hugepage" ON) option(ENABLE_TESTS "whether enable test" ON) @@ -31,9 +28,7 @@ option(ENABLE_3L_CACHE "enable 3LCache" OFF) set(LOG_LEVEL NONE CACHE STRING "change the logging level") set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVERBOSE VVVERBOSE) -# ####################################### -# detect platform # -# ####################################### +# Platform detection if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") message(STATUS "Mac OS X detected, version ${CMAKE_SYSTEM_VERSION}") add_definitions(-DOS_DARWIN) @@ -64,20 +59,10 @@ configure_file(libCacheSim/include/config.h.in libCacheSim/include/config.h) if(SUPPORT_TTL) add_compile_definitions(SUPPORT_TTL=1) -else() - remove_definitions(SUPPORT_TTL) -endif(SUPPORT_TTL) +endif() if(USE_HUGEPAGE) add_compile_definitions(USE_HUGEPAGE=1) -else() - remove_definitions(USE_HUGEPAGE) -endif(USE_HUGEPAGE) - -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/cache/eviction/priv") - add_compile_definitions(INCLUDE_PRIV=1) -else() - remove_definitions(INCLUDE_PRIV) endif() string(TOLOWER "${LOG_LEVEL}" LOG_LEVEL_LOWER) @@ -105,111 +90,108 @@ else() add_compile_definitions(LOGLEVEL=7) endif() -message(STATUS "CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG} CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO} CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}") - -# string( REPLACE "/DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}, LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}, ENABLE_LRB ${ENABLE_LRB}, ENABLE_3L_CACHE ${ENABLE_3L_CACHE}, OPT_SUPPORT_ZSTD_TRACE ${OPT_SUPPORT_ZSTD_TRACE}") +# Define shared compiler flags for all targets +set(LIBCACHESIM_C_FLAGS + -Wall -Wextra -Werror + -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable + -Wpedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings + -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -Wnested-externs -Wmissing-include-dirs +) -# add_compile_options(-fsanitize=address) -# add_link_options(-fsanitize=address) +set(LIBCACHESIM_CXX_FLAGS + -Wall -Wextra -Werror + -Wno-deprecated-copy -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable + -Wno-pedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wmissing-include-dirs +) -# ####################################### -# compiler flags # -# ####################################### -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wpedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -Wnested-externs -Wmissing-include-dirs") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-deprecated-copy -Wno-unused-variable -Wno-unused-function -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-pedantic -Wformat=2 -Wformat-security -Wshadow -Wwrite-strings -Wmissing-include-dirs") +# Find dependencies +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") -# if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") -# set(CFLAGS "$ENV{CFLAGS} " "-Wl,--export-dynamic ") -# endif() +# Include GNUInstallDirs for standard installation directories +include(GNUInstallDirs) -# string(REPLACE "" "" LOCAL_CFLAGS ${CFLAGS}) -# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} $ENV{CFLAGS} ${LOCAL_CFLAGS}") +set(dependency_libs "") -# ####################################### -# find dependency # -# ####################################### -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") +find_package(Threads REQUIRED) +set(dependency_libs ${dependency_libs} ${CMAKE_THREAD_LIBS_INIT}) find_package(GLib REQUIRED) include_directories(${GLib_INCLUDE_DIRS}) -set(LIBS ${LIBS} ${GLib_LIBRARY}) +set(dependency_libs ${dependency_libs} ${GLib_LIBRARY}) find_package(argp REQUIRED) include_directories(${ARGP_INCLUDE_DIRS}) -set(LIBS ${LIBS} ${ARGP_LIBRARY}) +set(dependency_libs ${dependency_libs} ${ARGP_LIBRARY}) if(OPT_SUPPORT_ZSTD_TRACE) add_compile_definitions(SUPPORT_ZSTD_TRACE=1) find_package(ZSTD) - # https://stackoverflow.com/questions/61377055/cannot-find-gflags-gflags-h-while-building-library-osx/61379123#61379123 include_directories(${ZSTD_INCLUDE_DIR}) if("${ZSTD_LIBRARIES}" STREQUAL "") message(FATAL_ERROR "zstd not found") endif() - link_libraries(${ZSTD_LIBRARIES}) + set(dependency_libs ${dependency_libs} ${ZSTD_LIBRARIES}) message(STATUS "ZSTD_INCLUDE_DIR ${ZSTD_INCLUDE_DIR}, ZSTD_LIBRARIES ${ZSTD_LIBRARIES}") -else() - remove_definitions(SUPPORT_ZSTD_TRACE) -endif(OPT_SUPPORT_ZSTD_TRACE) +endif() -# libgoogle-perftools-dev google-perftools -# tcmalloc causes trouble with valgrind https://github.com/gperftools/gperftools/issues/792 -# when using valgrind, we should not compile with tcmalloc -# maybe disable tcmalloc under debug model +# TCMalloc (libgoogle-perftools-dev google-perftools) +# Note: tcmalloc causes trouble with valgrind +# https://github.com/gperftools/gperftools/issues/792 +# When using valgrind, we should not compile with tcmalloc if(NOT ${CMAKE_BUILD_TYPE} MATCHES "Debug") find_package(Tcmalloc) if("${Tcmalloc_LIBRARY}" STREQUAL "") message(STATUS "!!! cannot find tcmalloc") else() - set(LIBS ${LIBS} ${Tcmalloc_LIBRARIES}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") + set(dependency_libs ${dependency_libs} ${Tcmalloc_LIBRARIES}) + # tcmalloc flags will be set per target instead of globally + set(tcmalloc_flags "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") endif() endif() find_package(Threads) -# find_package(Boost REQUIRED) -# message(STATUS "boost found? " ${Boost_FOUND} ", library " ${Boost_LIBRARIES} ", header " ${Boost_INCLUDE_DIRS}) -# include_directories(${Boost_INCLUDE_DIRS}) if(ENABLE_GLCACHE) find_package(xgboost REQUIRED) include_directories(${XGBOOST_INCLUDE_DIR}) - link_libraries(xgboost::xgboost) + set(dependency_libs ${dependency_libs} xgboost::xgboost) add_compile_definitions(ENABLE_GLCACHE=1) message(STATUS "XGBOOST_INCLUDE_DIR=${XGBOOST_INCLUDE_DIR}") -else() - remove_definitions(ENABLE_GLCACHE) endif() -foreach (FEATURE ENABLE_LRB ENABLE_3L_CACHE) - if (${FEATURE}) +foreach(FEATURE ENABLE_LRB ENABLE_3L_CACHE) + if(${FEATURE}) find_path(LIGHTGBM_PATH LightGBM) - if (NOT LIGHTGBM_PATH) + if(NOT LIGHTGBM_PATH) message(FATAL_ERROR "LIGHTGBM_PATH not found") - endif () + endif() include_directories(${LIGHTGBM_PATH}) find_library(LIGHTGBM_LIB _lightgbm) - if (NOT LIGHTGBM_LIB) + if(NOT LIGHTGBM_LIB) message(FATAL_ERROR "LIGHTGBM_LIB not found") - endif () - link_libraries(${LIGHTGBM_LIB}) + endif() + set(dependency_libs ${dependency_libs} ${LIGHTGBM_LIB}) add_compile_definitions(${FEATURE}=1) - else() - remove_definitions(${FEATURE}) endif() endforeach() -# link_libraries("-ldl -lm ${LIBS}") - -# put binary in bin directory +# Put binary in bin directory set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +# Status messages +message(STATUS "CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}") +message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") +message(STATUS "CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}") +message(STATUS "SUPPORT TTL ${SUPPORT_TTL}, USE_HUGEPAGE ${USE_HUGEPAGE}") +message(STATUS "LOGLEVEL ${LOG_LEVEL}, ENABLE_GLCACHE ${ENABLE_GLCACHE}") +message(STATUS "ENABLE_LRB ${ENABLE_LRB}, ENABLE_3L_CACHE ${ENABLE_3L_CACHE}") +message(STATUS "OPT_SUPPORT_ZSTD_TRACE ${OPT_SUPPORT_ZSTD_TRACE}") + message(STATUS "<<++=====------------------\\/------------------=====++>>") message(STATUS "<<++ libCacheSim summary ++>>") message(STATUS "<<++=====------------------/\\------------------=====++>>") @@ -226,156 +208,86 @@ message(STATUS "cmake source = ${CMAKE_CURRENT_SOURCE_DIR}") message(STATUS "cmake compiler = ${CMAKE_C_COMPILER}") message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}") message(STATUS "CFLAGS = ${CMAKE_C_FLAGS}") -message(STATUS "LIBS = ${LIBS}") +message(STATUS "dependency_libs = ${dependency_libs}") message(STATUS "Installation path = ${CMAKE_INSTALL_PREFIX}") message(STATUS "========================================================") message(STATUS "============= Status of optional features ==============") message(STATUS "========================================================") -# ####################################### -# library compilation # -# ####################################### -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/include) -set(ALL_MODULES cachelib admission prefetch evictionC evictionCPP traceReader profiler dataStructure ds_hash utils) +# Define include directories for reuse in subdirectories +set(libCacheSim_include_dir ${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/include ${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/utils/include ${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim) +set(libCacheSim_binary_include_dir ${CMAKE_CURRENT_BINARY_DIR}/libCacheSim/include) + +include_directories(${libCacheSim_include_dir}) +include_directories(${libCacheSim_binary_include_dir}) +set(all_modules cache_lib_c cache_lib_cpp traceReader_lib profiler_lib dataStructure_lib utils_lib traceAnalyzer_lib) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/cache) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/dataStructure) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/traceReader) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/profiler) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/utils) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/bin) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/traceAnalyzer) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/mrcProfiler) -# ####################################### -# library install # -# ####################################### -# compile a single library -file(GLOB LIB_SOURCE ${PROJECT_SOURCE_DIR}/libCacheSim/*.c) - -file(GLOB cache_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/admission/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/prefetch/*.c - - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/LHD/* - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/cpp/* -) - -if(EXISTS ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/fifo) - file(GLOB fifo_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/fifo/*.c) - set(cache_source - ${cache_source} ${fifo_source} - ) -endif() - -if(EXISTS ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/belady) - file(GLOB belady_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/belady/*.c) - set(cache_source - ${cache_source} ${belady_source} - ) -endif() +# libCacheSim binary / tool compilation +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/bin) -if(EXISTS ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/priv) - file(GLOB priv_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/priv/*.c) - set(cache_source - ${cache_source} ${priv_source} - ) +# libCacheSim test +if(ENABLE_TESTS) + include(CTest) + enable_testing() + message(STATUS "Building with test") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) +else() + message(STATUS "Building without test") endif() -if(ENABLE_GLCACHE) - file(GLOB GLCache_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/GLCache/*.c) - set(cache_source - ${cache_source} ${GLCache_source} - ) -endif(ENABLE_GLCACHE) - -if(ENABLE_LRB) - file(GLOB LRB_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/LRB/*.cpp) - set(cache_source - ${cache_source} ${LRB_source} - ) -endif(ENABLE_LRB) - -if(ENABLE_3L_CACHE) - file(GLOB ThreeLCache_source - ${PROJECT_SOURCE_DIR}/libCacheSim/cache/eviction/3LCache/*.cpp) - set(cache_source - ${cache_source} ${ThreeLCache_source} - ) -endif(ENABLE_3L_CACHE) - -set(reader_source - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/reader.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/binary.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/csv.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/customizedReader/lcs.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/libcsv.c - ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/txt.c -) -if(OPT_SUPPORT_ZSTD_TRACE) - set(reader_source - ${reader_source} ${PROJECT_SOURCE_DIR}/libCacheSim/traceReader/generalReader/zstdReader.c - ) -endif(OPT_SUPPORT_ZSTD_TRACE) - -file(GLOB dataStructure_source - ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hashtable/*.c - ${PROJECT_SOURCE_DIR}/libCacheSim/dataStructure/hash/murmur3.c -) +# libCacheSim unified library compilation and installation +# Create a single library that combines all modular libraries +add_library(${PROJECT_NAME} INTERFACE) -file(GLOB profiler_source - ${PROJECT_SOURCE_DIR}/libCacheSim/profiler/*.c -) +# Link all modular libraries into the unified library +target_link_libraries(${PROJECT_NAME} INTERFACE ${all_modules}) -file(GLOB utils_source - ${PROJECT_SOURCE_DIR}/libCacheSim/utils/*.c +# Set include directories for the unified library +target_include_directories(${PROJECT_NAME} INTERFACE + $ + $ + $ ) -set(LIB_SOURCE ${LIB_SOURCE} ${cache_source} ${reader_source} ${dataStructure_source} ${profiler_source} ${utils_source}) - -# ################## -# # https://stackoverflow.com/questions/32469953/why-is-cmake-designed-so-that-it-removes-runtime-path-when-installing -SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - -add_library(${PROJECT_NAME} ${LIB_SOURCE}) -# target_compile_options(${PROJECT_NAME} PRIVATE -fPIC) -target_link_options(${PROJECT_NAME} PRIVATE -Wl,--export-dynamic) +# Link dependencies for the unified library +target_link_libraries(${PROJECT_NAME} INTERFACE ${dependency_libs}) - -# add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE}) +# Set version and properties set_target_properties(${PROJECT_NAME} PROPERTIES VERSION ${${PROJECT_NAME}_VERSION}) -set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C) -set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/libCacheSim/include/libCacheSim.h) +# Configuration files configure_file(${PROJECT_SOURCE_DIR}/${PROJECT_NAME}.pc.in ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc @ONLY) configure_file(${PROJECT_SOURCE_DIR}/${PROJECT_NAME}.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/Find${PROJECT_NAME}.cmake @ONLY) -# this overwrites the default config.h -# configure_file(${PROJECT_SOURCE_DIR}/libCacheSim/include/config.h.in ${PROJECT_SOURCE_DIR}/libCacheSim/include/config.h @ONLY) -install(DIRECTORY ${PROJECT_SOURCE_DIR}/libCacheSim/include/ DESTINATION include) -install(FILES ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc DESTINATION lib/pkgconfig) -install(FILES ${CMAKE_BINARY_DIR}/Find${PROJECT_NAME}.cmake DESTINATION ${CMAKE_ROOT}/Modules/ COMPONENT dev) - -install(TARGETS ${PROJECT_NAME} +# Install configuration +install(DIRECTORY ${PROJECT_SOURCE_DIR}/libCacheSim/include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/Find${PROJECT_NAME}.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} COMPONENT dev) + +# Install all modular libraries and the unified interface library +install(TARGETS ${PROJECT_NAME} ${all_modules} + EXPORT ${PROJECT_NAME}Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +# Install the export file +install(EXPORT ${PROJECT_NAME}Targets + FILE ${PROJECT_NAME}Targets.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} +) -# ############################ -if(ENABLE_TESTS) - include(CTest) - enable_testing() - message(STATUS "Building with test") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) -else() - message(STATUS "Building without test") -endif() diff --git a/libCacheSim/bin/CMakeLists.txt b/libCacheSim/bin/CMakeLists.txt index 0fd39af3..abcb7508 100644 --- a/libCacheSim/bin/CMakeLists.txt +++ b/libCacheSim/bin/CMakeLists.txt @@ -1,8 +1,9 @@ - +# ============================================================================== +# Binaries +# ============================================================================== add_subdirectory(MRC) add_subdirectory(cachesim) -# add_subdirectory(traceWriter) add_subdirectory(distUtil) add_subdirectory(traceUtils) add_subdirectory(traceAnalyzer) @@ -10,12 +11,5 @@ add_subdirectory(mrcProfiler) add_subdirectory(debug) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/customized) - message(STATUS "Found customized directory, building customized") add_subdirectory(customized/SOSP23) endif() - -if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/priv) - message(STATUS "Found priv directory, building priv") - add_subdirectory(priv) -endif() - diff --git a/libCacheSim/bin/MRC/CMakeLists.txt b/libCacheSim/bin/MRC/CMakeLists.txt index 25cf407d..10d269c1 100644 --- a/libCacheSim/bin/MRC/CMakeLists.txt +++ b/libCacheSim/bin/MRC/CMakeLists.txt @@ -1,5 +1,19 @@ +# ============================================================================== +# MRC +# ============================================================================== -add_executable(MRC main.c parser_shard.c SHARDS.c ../cli_reader_utils.c ../../dataStructure/histogram.c ../../dataStructure/splay_tuple.c ../../dataStructure/splay.c parser_mini.c Miniatures.c) -target_link_libraries(MRC ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) -install(TARGETS MRC RUNTIME DESTINATION bin) +set(mrc_sources_c + main.c + parser_shard.c + SHARDS.c + ../cli_reader_utils.c + ../../dataStructure/histogram.c + ../../dataStructure/splay_tuple.c + ../../dataStructure/splay.c + parser_mini.c + Miniatures.c +) +add_executable(MRC ${mrc_sources_c}) +target_link_libraries(MRC ${all_modules} ${dependency_libs}) +install(TARGETS MRC RUNTIME DESTINATION bin) diff --git a/libCacheSim/bin/cachesim/CMakeLists.txt b/libCacheSim/bin/cachesim/CMakeLists.txt index 3c872b0e..de3487a1 100644 --- a/libCacheSim/bin/cachesim/CMakeLists.txt +++ b/libCacheSim/bin/cachesim/CMakeLists.txt @@ -1,7 +1,14 @@ +# ============================================================================== +# Cachesim +# ============================================================================== -add_executable(cachesim main.c cli_parser.c sim.c ../cli_reader_utils.c) -target_link_libraries(cachesim ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +set(cachesim_sources_c + main.c + cli_parser.c + sim.c + ../cli_reader_utils.c +) +add_executable(cachesim ${cachesim_sources_c}) +target_link_libraries(cachesim ${all_modules} ${dependency_libs}) +target_link_options(cachesim PRIVATE "-Wl,--export-dynamic") install(TARGETS cachesim RUNTIME DESTINATION bin) - -# add_executable(flash flash.cpp cli_parser.c sim.c ../cli_reader_utils.c) -# target_link_libraries(flash ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) diff --git a/libCacheSim/bin/cachesim/cache_init.h b/libCacheSim/bin/cachesim/cache_init.h index 339e9efc..cc47d4bf 100644 --- a/libCacheSim/bin/cachesim/cache_init.h +++ b/libCacheSim/bin/cachesim/cache_init.h @@ -32,52 +32,44 @@ static inline cache_t *create_cache(const char *trace_path, const char *name; cache_t *(*init_func)(common_cache_params_t, const char *); } eviction_algo_entry_t; - static const eviction_algo_entry_t simple_algos[] = { - {"lru", LRU_init}, - {"fifo", FIFO_init}, + {"2q", TwoQ_init}, {"arc", ARC_init}, {"arcv0", ARCv0_init}, - {"lhd", LHD_init}, - {"random", Random_init}, - {"randomTwo", RandomTwo_init}, - {"lfu", LFU_init}, - {"gdsf", GDSF_init}, - {"lfuda", LFUDA_init}, - {"twoq", TwoQ_init}, - {"2q", TwoQ_init}, - {"slru", SLRU_init}, - {"slruv0", SLRUv0_init}, - {"lecar", LeCaR_init}, - {"lecarv0", LeCaRv0_init}, - {"RandomLRU", RandomLRU_init}, + {"CAR", CAR_init}, {"cacheus", Cacheus_init}, - {"size", Size_init}, - {"lfucpp", LFUCpp_init}, - {"wtinyLFU", WTinyLFU_init}, - {"nop", nop_init}, - {"fifo-reinsertion", Clock_init}, {"clock", Clock_init}, - {"second-chance", Clock_init}, {"clockpro", ClockPro_init}, - {"lirs", LIRS_init}, - {"fifomerge", FIFO_Merge_init}, + {"fifo", FIFO_init}, {"fifo-merge", FIFO_Merge_init}, + {"fifo-reinsertion", Clock_init}, + {"fifomerge", FIFO_Merge_init}, {"flashProb", flashProb_init}, - {"sfifo", SFIFO_init}, - {"sfifov0", SFIFOv0_init}, + {"gdsf", GDSF_init}, + {"lhd", LHD_init}, + {"lecar", LeCaR_init}, + {"lecarv0", LeCaRv0_init}, + {"lfu", LFU_init}, + {"lfucpp", LFUCpp_init}, + {"lfuda", LFUDA_init}, + {"lirs", LIRS_init}, + {"lru", LRU_init}, {"lru-prob", LRU_Prob_init}, - {"fifo-belady", FIFO_Belady_init}, - {"lru-belady", LRU_Belady_init}, - {"sieve-belady", Sieve_Belady_init}, - {"s3lru", S3LRU_init}, - {"s3fifo", S3FIFO_init}, + {"nop", nop_init}, + {"qdlp", QDLP_init}, + {"random", Random_init}, + {"RandomLRU", RandomLRU_init}, + {"randomTwo", RandomTwo_init}, {"s3-fifo", S3FIFO_init}, - {"s3fifov0", S3FIFOv0_init}, {"s3-fifov0", S3FIFOv0_init}, + {"s3fifo", S3FIFO_init}, {"s3fifod", S3FIFOd_init}, - {"qdlp", QDLP_init}, - {"CAR", CAR_init}, + {"s3fifov0", S3FIFOv0_init}, + {"size", Size_init}, + {"slru", SLRU_init}, + {"slruv0", SLRUv0_init}, + {"twoq", TwoQ_init}, + {"wtinyLFU", WTinyLFU_init}, #ifdef ENABLE_3L_CACHE {"3LCache", ThreeLCache_init}, #endif @@ -87,15 +79,6 @@ static inline cache_t *create_cache(const char *trace_path, #endif #ifdef ENABLE_LRB {"lrb", LRB_init}, -#endif -#ifdef INCLUDE_PRIV - {"mclock", MClock_init}, - {"lp-sfifo", LP_SFIFO_init}, - {"lp-arc", LP_ARC_init}, - {"lp-twoq", LP_TwoQ_init}, - {"qdlpv0", QDLPv0_init}, - {"s3fifodv2", S3FIFOdv2_init}, - {"myMQv1", myMQv1_init} #endif }; diff --git a/libCacheSim/bin/cachesim/filter.c b/libCacheSim/bin/cachesim/filter.c deleted file mode 100644 index 68bcd22a..00000000 --- a/libCacheSim/bin/cachesim/filter.c +++ /dev/null @@ -1,83 +0,0 @@ - - -#include - -#include "../../include/libCacheSim/cache.h" -#include "../../include/libCacheSim/reader.h" -#include "../../utils/include/mymath.h" -#include "../../utils/include/mystr.h" -#include "../../utils/include/mysys.h" -#include "internal.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define REPORT_INTERVAL (24 * 3600) - -struct output_format { - uint32_t clock_time; - uint64_t obj_id; - uint32_t obj_size; - int64_t next_access_vtime; -} __attribute__((packed)); - -void filter(reader_t *reader, cache_t *cache) { - /* random seed */ - srand(time(NULL)); - set_rand_seed(rand()); - - char ofilepath[128]; - sprintf(ofilepath, "%s.filter%s", basename(reader->trace_path), - cache->cache_name); - request_t *req = new_request(); - - FILE *output_file = fopen(ofilepath, "wb"); - struct output_format output_req; - output_req.next_access_vtime = -2; - - read_one_req(reader, req); - uint64_t start_ts = (uint64_t)req->clock_time; - - int64_t n_req = 0, n_written_req = 0; - while (req->valid) { - n_req++; - req->clock_time -= start_ts; - if (cache->get(cache, req) == false) { - output_req.clock_time = req->clock_time; - output_req.obj_id = req->obj_id; - output_req.obj_size = req->obj_size; - fwrite(&output_req, sizeof(struct output_format), 1, output_file); - n_written_req++; - } - - read_one_req(reader, req); - } - - INFO("write %ld/%ld %.4lf requests to %s file\n", n_written_req, n_req, - (double)n_written_req / n_req, ofilepath); - fclose(output_file); -} - -int main(int argc, char *argv[]) { - struct arguments args; - parse_cmd(argc, argv, &args); - if (args.n_cache_size != 1) { - WARN("only support one cache size\n"); - exit(0); - } - if (args.n_eviction_algo != 1) { - WARN("only support one eviction algorithm\n"); - exit(0); - } - - filter(args.reader, args.caches[0]); - - free_arg(&args); - - return 0; -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/bin/customized/SOSP23/CMakeLists.txt b/libCacheSim/bin/customized/SOSP23/CMakeLists.txt index 11b29555..1c2734eb 100644 --- a/libCacheSim/bin/customized/SOSP23/CMakeLists.txt +++ b/libCacheSim/bin/customized/SOSP23/CMakeLists.txt @@ -1,8 +1,20 @@ +# ============================================================================== +# SOSP23 +# ============================================================================== -add_library(traceOneHitLib oneHit/oneHit.cpp) -add_executable(traceOneHit oneHit/main.c oneHit/oneHit.cpp oneHit/cli_parser.c ../../cli_reader_utils.c) -target_link_libraries(traceOneHit traceOneHitLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +set(traceOneHit_sources_cpp + oneHit/oneHit.cpp + oneHit/cli_parser.c + oneHit/main.c + ../../cli_reader_utils.c +) +add_executable(traceOneHit ${traceOneHit_sources_cpp}) +target_link_libraries(traceOneHit dataStructure_lib traceReader_lib ${Boost_LIBRARIES} ${dependency_libs}) - -add_executable(flash flash/flash.cpp ../../cachesim/cli_parser.c ../../cachesim/sim.c ../../cli_reader_utils.c) -target_link_libraries(flash ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +set(flash_sources_cpp + flash/flash.cpp + oneHit/cli_parser.c + ../../cli_reader_utils.c +) +add_executable(flash ${flash_sources_cpp}) +target_link_libraries(flash ${all_modules} utils_lib ${dependency_libs}) diff --git a/libCacheSim/bin/customized/fast23/CMakeLists.txt b/libCacheSim/bin/customized/fast23/CMakeLists.txt index 2ff996aa..096c2f0f 100644 --- a/libCacheSim/bin/customized/fast23/CMakeLists.txt +++ b/libCacheSim/bin/customized/fast23/CMakeLists.txt @@ -1,4 +1,12 @@ +# ============================================================================== +# Fast23 +# ============================================================================== - -add_executable(fast23 main.cpp compareGrouping.cpp objectInfo.cpp compareGroups.cpp) -target_link_libraries(fast23 analysisLib dataStructureLib traceReader ${Boost_LIBRARIES}) +set(fast23_sources_cpp + main.cpp + compareGrouping.cpp + objectInfo.cpp + compareGroups.cpp +) +add_executable(fast23 ${fast23_sources_cpp}) +target_link_libraries(fast23 dataStructure_lib traceReader_lib ${Boost_LIBRARIES}) diff --git a/libCacheSim/bin/customized/traceWriter/CMakeLists.txt b/libCacheSim/bin/customized/traceWriter/CMakeLists.txt deleted file mode 100644 index 4310a347..00000000 --- a/libCacheSim/bin/customized/traceWriter/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ - - -add_executable(traceWriter main.c) -target_link_libraries(traceWriter ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) - diff --git a/libCacheSim/bin/customized/traceWriter/main.c b/libCacheSim/bin/customized/traceWriter/main.c deleted file mode 100644 index 4645283c..00000000 --- a/libCacheSim/bin/customized/traceWriter/main.c +++ /dev/null @@ -1,122 +0,0 @@ - - -#include -#include -#include -#include -#include - -#include "../../include/libCacheSim/reader.h" - -void fix_msr_oracleGeneral_trace(char *trace_path); -void fix_msr_oracleGeneralOpNS_trace(char *trace_path); -void convert_wiki16_trace(char *trace_path); - -int main(int argc, char *argv[]) { - // fix_msr_oracleGeneral_trace(argv[1]); - // fix_msr_oracleGeneralOpNS_trace(argv[1]); - - convert_wiki16_trace(argv[1]); - - return 0; -} - -void fix_msr_oracleGeneral_trace(char *trace_path) { - reader_t *reader = - setup_reader(trace_path, ORACLE_GENERAL_TRACE, NULL); - request_t *req = new_request(); - - char ofilepath[256]; - sprintf(ofilepath, "%s.new", trace_path); - FILE *ofile = fopen(ofilepath, "wb"); - read_one_req(reader, req); - uint64_t start_ts = req->clock_time; - - while (req->valid) { - req->clock_time = (req->clock_time - start_ts) / 10; - if (req->next_access_vtime == INT64_MAX) { - req->next_access_vtime = -1; - } - - fwrite(&req->clock_time, 4, 1, ofile); - fwrite(&req->obj_id, 8, 1, ofile); - fwrite(&req->obj_size, 4, 1, ofile); - fwrite(&req->next_access_vtime, 8, 1, ofile); - - read_one_req(reader, req); - } - printf("%s trace time %lu hour\n", trace_path, req->clock_time / 3600); - - fclose(ofile); -} - -void fix_msr_oracleGeneralOpNS_trace(char *trace_path) { - reader_t *reader = - setup_reader(trace_path, ORACLE_GENERALOPNS_TRACE, NULL); - request_t *req = new_request(); - - char ofilepath[256]; - sprintf(ofilepath, "%s.new", trace_path); - FILE *ofile = fopen(ofilepath, "wb"); - // char ofilepath2[256]; - // sprintf(ofilepath2, "%s.new.txt", trace_path); - // FILE *ofile2 = fopen(ofilepath2, "w"); - read_one_req(reader, req); - uint64_t start_ts = req->clock_time; - - while (req->valid) { - // req->clock_time = (req->clock_time - start_ts) / 10; - // if (req->next_access_vtime == INT64_MAX) { - // req->next_access_vtime = -1; - // } - - uint8_t op = req->op; - uint16_t ns = req->ns; - - fwrite(&req->clock_time, 4, 1, ofile); - fwrite(&req->obj_id, 8, 1, ofile); - fwrite(&req->obj_size, 4, 1, ofile); - fwrite(&op, 1, 1, ofile); - fwrite(&ns, 2, 1, ofile); - fwrite(&req->next_access_vtime, 8, 1, ofile); - - // fprintf(ofile2, "%lu %lu %u %lu %u %u\n", req->clock_time, req->obj_id, - // req->obj_size, req->next_access_vtime, op, ns); - - read_one_req(reader, req); - } - printf("%s trace time %lu hour\n", trace_path, req->clock_time / 3600); - - fclose(ofile); -} - -void convert_wiki16_trace(char *trace_path) { - reader_t *reader = - setup_reader(trace_path, ORACLE_WIKI16u_TRACE, NULL); - request_t *req = new_request(); - uint64_t n = 0; - uint64_t n_total_req = get_num_of_req(reader); - - char ofilepath[256]; - sprintf(ofilepath, "%s.new", trace_path); - FILE *ofile = fopen(ofilepath, "wb"); - read_one_req(reader, req); - - while (req->valid) { - req->clock_time = (int)((double)n / (double)n_total_req * 3600 * 168); - if (req->next_access_vtime == INT64_MAX) { - req->next_access_vtime = -1; - } - - n += 1; - fwrite(&req->clock_time, 4, 1, ofile); - fwrite(&req->obj_id, 8, 1, ofile); - fwrite(&req->obj_size, 4, 1, ofile); - fwrite(&req->next_access_vtime, 8, 1, ofile); - - read_one_req(reader, req); - } - printf("%s trace time %lu hour\n", trace_path, req->clock_time / 3600); - - fclose(ofile); -} \ No newline at end of file diff --git a/libCacheSim/bin/debug/CMakeLists.txt b/libCacheSim/bin/debug/CMakeLists.txt index 59a31ab2..68f5b651 100644 --- a/libCacheSim/bin/debug/CMakeLists.txt +++ b/libCacheSim/bin/debug/CMakeLists.txt @@ -1,9 +1,13 @@ +# ============================================================================== +# Debug +# ============================================================================== add_executable(debug main.cpp) -target_link_libraries(debug ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +target_link_libraries(debug ${all_modules} ${dependency_libs}) add_executable(debug_aligned aligned.c) -target_link_libraries(debug_aligned ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +target_link_libraries(debug_aligned ${all_modules} ${dependency_libs}) add_executable(debug_fileOp fileOp.cpp) -target_link_libraries(debug_fileOp ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +target_link_libraries(debug_fileOp ${all_modules} ${dependency_libs}) + diff --git a/libCacheSim/bin/dep/CMakeLists.txt b/libCacheSim/bin/dep/CMakeLists.txt deleted file mode 100644 index 241c8b25..00000000 --- a/libCacheSim/bin/dep/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ - - -add_subdirectory(analysis) -add_subdirectory(oracleTraceGen) -add_subdirectory(debug) -add_subdirectory(fast23) - diff --git a/libCacheSim/bin/dep/cpp/CMakeLists.txt b/libCacheSim/bin/dep/cpp/CMakeLists.txt deleted file mode 100644 index 541416ed..00000000 --- a/libCacheSim/bin/dep/cpp/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -cmake_minimum_required(VERSION 3.2) -project(traceUtil) -set(DESCRIPTION "a simple trace utility") - -set(${PROJECT_NAME}_VERSION_MAJOR 0) -set(${PROJECT_NAME}_VERSION_MINOR 1) -set(${PROJECT_NAME}_VERSION_PATCH 0) -set(${PROJECT_NAME}_RELEASE_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}) -set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_RELEASE_VERSION}.${${PROJECT_NAME}_VERSION_PATCH}) - - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - - -set(CFLAGS "$ENV{CFLAGS} " - "-Wall -Wshadow -Winline " - "-Wno-unused " - "-Wredundant-decls " - ) - -string(REPLACE "" "" LOCAL_CFLAGS ${CFLAGS}) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} $ENV{CFLAGS} ${LOCAL_CFLAGS}") - - -if (CMAKE_BUILD_TYPE MATCHES Debug) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g -ggdb3 -DLOGLEVEL=5 ") - message(STATUS "Debug build") -else () - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3") - message(STATUS "Release build") -endif () - - -add_executable(conv traceConv.cpp) - -add_executable(split traceSplit.cpp) diff --git a/libCacheSim/bin/dep/cpp/README b/libCacheSim/bin/dep/cpp/README deleted file mode 100644 index 24fb880c..00000000 --- a/libCacheSim/bin/dep/cpp/README +++ /dev/null @@ -1,3 +0,0 @@ - -a simple cpp script to convert traces from text to binary, used internally for alibaba and tencent, check libCacheSim for general-purpose trace converter - diff --git a/libCacheSim/bin/dep/cpp/alibaba.h b/libCacheSim/bin/dep/cpp/alibaba.h deleted file mode 100644 index 677a973c..00000000 --- a/libCacheSim/bin/dep/cpp/alibaba.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once - -#include "traceConv.h" - -static inline void parse_alibaba(std::string line, struct trace_req *req) { - // ns, op, lba, sz, ts = line.strip().split(",") - uint64_t lba; - - char *p = strtok(line.data(), ","); - req->ns = (uint16_t) atoi(p); - - p = strtok(NULL, ","); - switch (*p) { - case 'R': - req->op = 1; - break; - case 'W': - req->op = 3; - break; - default: - std::cerr << "unknown op: " << *p << std::endl; - req->op = 0; - break; - } - - p = strtok(NULL, ","); - lba = atoll(p); - req->obj_id = lba + req->ns; - p = strtok(NULL, ","); - req->sz = atoi(p); - p = strtok(NULL, ","); - req->ts = atoll(p) / 1000000; - p = strtok(NULL, ","); - assert(p == NULL); -} - diff --git a/libCacheSim/bin/dep/cpp/common.h b/libCacheSim/bin/dep/cpp/common.h deleted file mode 100644 index 4de76616..00000000 --- a/libCacheSim/bin/dep/cpp/common.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#include - - -struct trace_req { - uint32_t ts; - uint64_t obj_id; - uint32_t sz; - uint8_t op; - uint32_t ns; // namespace -} __attribute__((packed)); - - -// enum op { -// OP_INVALID = 0, -// OP_READ = 1, -// OP_WRITE = 3, -// OP_UNKNOWN = 4 -// }; - -enum output_trace_type { - standardIQI = 1, - standardIQIbh = 2, // opNS -}; diff --git a/libCacheSim/bin/dep/cpp/tencent.h b/libCacheSim/bin/dep/cpp/tencent.h deleted file mode 100644 index 0685ca71..00000000 --- a/libCacheSim/bin/dep/cpp/tencent.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include - -#include "traceConv.h" - -static inline void parse_tencent_block(std::string line, - struct trace_req *req) { - // # 1538323199,105352008,584,1,1576 - - uint64_t lba; - - char *p = strtok(line.data(), ","); - req->ts = (uint32_t) atoi(p); - p = strtok(NULL, ","); - lba = atoll(p); - p = strtok(NULL, ","); - req->sz = atoi(p) * 512; - - p = strtok(NULL, ","); - if (*p == '0') { - req->op = 1; - } else if (*p == '1') { - req->op = 3; - } else { - std::cerr << "unknown op: " << *p << std::endl; - req->op = 0; - } - - p = strtok(NULL, ","); - req->ns = (uint16_t) atoi(p); - - req->obj_id = lba + req->ns; - p = strtok(NULL, ","); - assert(p == NULL); -} - -static inline void parse_tencent_photo(std::string line, - struct trace_req *req) { - // 20160201164559 001488a9e71a3064716358a6e62a99344068d7bd 0 m 80234 1 PHONE 0 - // ts, obj_id, _, _, sz, _, _, _ = line.strip().split() - - struct tm tm = {}; - - char *p = strtok(line.data(), " "); - - strptime(p, "%Y%m%d%H%M%S", &tm); - req->ts = (uint32_t) std::mktime(&tm); - - p = strtok(NULL, " "); - req->obj_id = (int64_t) std::hash{}(p); - - p = strtok(NULL, " "); - p = strtok(NULL, " "); - p = strtok(NULL, " "); - req->sz = atol(p); - - p = strtok(NULL, " "); - p = strtok(NULL, " "); - p = strtok(NULL, " "); - p = strtok(NULL, " "); - assert(p == NULL); -} \ No newline at end of file diff --git a/libCacheSim/bin/dep/cpp/traceConv.cpp b/libCacheSim/bin/dep/cpp/traceConv.cpp deleted file mode 100644 index 8823eedf..00000000 --- a/libCacheSim/bin/dep/cpp/traceConv.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/** - * convert text traces to binary traces - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "alibaba.h" -#include "tencent.h" -#include "traceConv.h" - -using namespace std; - -const size_t BUFFER_SIZE = 200000; - -struct trace_req_opns { - uint32_t ts; - uint64_t obj_id; - uint32_t sz; - uint8_t op; - uint32_t ns; // namespace -} __attribute__((packed)); - -int main(int argc, char *argv[]) { - if (argc < 4) { - cout << "convert txt/csv trace to standardBinIQI/standardBinIQIbh\n" - << "standardBinIQI: ts, obj_id, sz\n" - << "standardBinIQIbh: ts, obj_id, sz, op, ns\n" - << "Usage: ./traceConv trace_file trace_type " - << "(alibaba/tencentBlock/tencentPhoto) format(IQI/IQIbh) per_ns (0/1)" - << endl; - return 0; - } - - string trace_file = argv[1]; - string trace_type = argv[2]; - string format = argv[3]; - bool per_ns = atoi(argv[4]); - - ifstream ifs(trace_file, ios::in); - if (!ifs.is_open()) { - cout << "open trace file failed" << endl; - return 0; - } - - string output_filepath = trace_file + ".bin." + format; - ofstream ofs(output_filepath, ios::out | ios::binary); - if (!ofs.is_open()) { - cout << "open output file " + output_filepath + " failed" << endl; - return 0; - } - - parse_func_t *parse_func = NULL; - if (trace_type == "alibaba") { - parse_func = parse_alibaba; - } else if (trace_type == "tencentBlock") { - parse_func = parse_tencent_block; - } else if (trace_type == "tencentPhoto") { - parse_func = parse_tencent_photo; - } else { - cout << "trace type not supported" << endl; - return 0; - } - - enum output_trace_type output_trace_type = standardIQI; - size_t write_size = 0; - if (format == "IQI") { - output_trace_type = standardIQI; - write_size = sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t); - } else if (format == "IQIbh") { - output_trace_type = standardIQIbh; - write_size = sizeof(uint32_t) + sizeof(uint64_t) + sizeof(uint32_t) + - sizeof(uint8_t) + sizeof(uint16_t); - } else { - cout << "format not supported" << endl; - return 0; - } - - uint64_t n = 0; - auto proc_start = chrono::system_clock::now(); - uint32_t trace_start_ts = UINT32_MAX; - string line; - struct trace_req req; - struct trace_req_opns req_opns; - unordered_map ofs_map; // ns -> ofs - unordered_map> - buffer_map; // ns -> per ns buffer - - while (getline(ifs, line)) { - if (++n % 100000000 == 0) { - chrono::duration elapsed_seconds = - chrono::system_clock::now() - proc_start; - cout << elapsed_seconds.count() << "s: " << n / 1000000 << "M req, " - << buffer_map.size() << " namespaces" << endl; - } - - parse_func(line, &req); - if (trace_start_ts == UINT32_MAX) { - trace_start_ts = req.ts; - } - req.ts -= trace_start_ts; - - ofs.write((char *)&req, write_size); - - int ns = req.ns; - if (per_ns) { - buffer_map[req.ns].push_back(req); - if (buffer_map[req.ns].size() >= BUFFER_SIZE) { - if (ofs_map.find(req.ns) == ofs_map.end()) { - string ofs_name = output_filepath + "." + to_string(req.ns); - ofstream f(ofs_name, ios::out | ios::binary); - if (!f) { - cerr << "open " + ofs_name + " failed, check ulimit" << endl; - abort(); - } else { - ofs_map[req.ns] = std::move(f); - } - } - - for (auto &req : buffer_map[req.ns]) { - if (format == "IQI") { - ofs_map[req.ns].write((char *)&req, write_size); - } else if (format == "IQIbh") { - req_opns.ts = req.ts; - req_opns.obj_id = req.obj_id; - req_opns.sz = req.sz; - req_opns.op = req.op; - if (req.ns > 65535) { - cout << "ns " << req.ns << " too large" << endl; - abort(); - } - req_opns.ns = req.ns; - ofs_map[req.ns].write((char *)&req_opns, write_size); - } - assert(req.ns == ns); // ns should be the same - } - buffer_map[req.ns].clear(); - } - } - } - - for (auto &t : buffer_map) { - if (ofs_map.find(t.first) != ofs_map.end()) { - for (auto &req : t.second) { - assert(req.ns == t.first); - ofs_map[req.ns].write((char *)&req, write_size); - } - } - t.second.clear(); - } - - ifs.close(); - ofs.close(); - for (auto it = ofs_map.begin(); it != ofs_map.end(); it++) { - it->second.close(); - } - - return 0; -} diff --git a/libCacheSim/bin/dep/cpp/traceConv.h b/libCacheSim/bin/dep/cpp/traceConv.h deleted file mode 100644 index 8574549f..00000000 --- a/libCacheSim/bin/dep/cpp/traceConv.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include -#include - -#include "common.h" - - -typedef void(parse_func_t)(std::string line, struct trace_req *req); diff --git a/libCacheSim/bin/dep/cpp/traceSplit.cpp b/libCacheSim/bin/dep/cpp/traceSplit.cpp deleted file mode 100644 index f9d1ac35..00000000 --- a/libCacheSim/bin/dep/cpp/traceSplit.cpp +++ /dev/null @@ -1,212 +0,0 @@ -/** - * split traces by tenants - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" - -using namespace std; - -const size_t BUFFER_SIZE = 1000000; - -typedef int(read_func_t)(ifstream &ifs, struct trace_req *req); - -int read_oracleCF1_trace(ifstream &ifs, struct trace_req *req) { - /* ts), sizeof(uint32_t)).eof()) { - return 1; - } - ifs.read((char *)&(req->obj_id), sizeof(uint64_t)); - ifs.read((char *)&sz, sizeof(uint64_t)); - ifs.read((char *)&u32, sizeof(uint32_t)); // ttl - ifs.read((char *)&u32, sizeof(uint32_t)); // age - ifs.read((char *)&(req->ns), sizeof(uint32_t)); // hostname - ifs.read((char *)&u64, sizeof(uint64_t)); // next_access_vtime - ifs.read((char *)&u16, sizeof(uint16_t)); // content - ifs.read((char *)&u16, sizeof(uint16_t)); // extension - ifs.read((char *)&u16, sizeof(uint16_t)); // n_level - ifs.read((char *)&u8, sizeof(uint8_t)); // n_param - ifs.read((char *)&u8, sizeof(uint8_t)); // method - ifs.read((char *)&u8, sizeof(uint8_t)); // colo - req->sz = (uint32_t)sz; - - if (sz > UINT32_MAX) { - if (sz > 0xFFFFFFFF00000000) { - sz -= 0xFFFFFFFF00000000; - } - req->sz = UINT32_MAX; - } - - return 0; -} - -int read_oracleAkamai_trace(ifstream &ifs, struct trace_req *req) { - /* - req->real_time = *(uint32_t *) record; - req->obj_id = *(uint64_t *) (record + 4); - req->obj_size = *(uint32_t *) (record + 12); - req->tenant_id = *(int16_t *) (record + 16) - 1; - req->bucket_id = *(int16_t *) (record + 18) - 1; - req->content_type = *(int16_t *) (record + 20) - 1; - req->next_access_vtime = *(int64_t *) (record + 22); - */ - - uint64_t u64; - uint32_t sz; - uint32_t u32; - uint16_t u16; - - if (ifs.read((char *)&(req->ts), sizeof(uint32_t)).eof()) { - return 1; - } - ifs.read((char *)&(req->obj_id), sizeof(uint64_t)); - ifs.read((char *)&(req->sz), sizeof(uint32_t)); - ifs.read((char *)&(req->ns), sizeof(uint16_t)); // ns - ifs.read((char *)&u16, sizeof(uint16_t)); // bucket - ifs.read((char *)&u16, sizeof(uint16_t)); // content - ifs.read((char *)&u64, sizeof(uint64_t)); // next_access_vtime - - if (sz > UINT32_MAX) { - cout << "size too large " << sz << " > " << UINT32_MAX << endl; - printf("read %u %lu %u %u\n", req->ts, req->obj_id, sz, req->ns); - - if (sz > 0xFFFFFFFF00000000) { - sz -= 0xFFFFFFFF00000000; - } - - req->sz = UINT32_MAX; - } - - return 0; -} - -int main(int argc, char *argv[]) { - cout << "split the trace by tenants, output is standardBinIQI" << endl; - if (argc < 2) { - cout << "Usage: " << argv[0] << " trace_file trace_type (oracleCF1/oracleAkamai) ofilepath" << endl; - return 0; - } - - string trace_file = argv[1]; - string trace_type = std::string(argv[2]); - string ofilepath = argv[3]; - - boost::algorithm::to_lower(trace_type); - - ifstream ifs(trace_file, ios::in | ios::binary); - if (!ifs.is_open()) { - cout << "open trace file failed" << endl; - return 0; - } - - read_func_t *read_func = NULL; - if (trace_type == "oraclecf1") { - read_func = read_oracleCF1_trace; - } else if (trace_type == "oracleakamai") { - read_func = read_oracleAkamai_trace; - } else { - cout << "trace type " << trace_type << " not supported" << endl; - return 0; - } - - /* ns -> ofs */ - unordered_map ofs_map; - /* ns -> per ns buffer */ - unordered_map> buffer_map; - - auto proc_start = chrono::system_clock::now(); - struct trace_req req; - uint32_t trace_start_ts = UINT32_MAX; - uint64_t n = 0; - - while (read_func(ifs, &req) == 0) { - if (++n % 100000000 == 0) { - chrono::duration elapsed_sec = - chrono::system_clock::now() - proc_start; - cout << elapsed_sec.count() << "s: " << n / 1000000 << "M req, " - << buffer_map.size() << "/" << ofs_map.size() << " namespaces" - << endl; - } - - if (trace_start_ts == UINT32_MAX) { - trace_start_ts = req.ts; - } - req.ts -= trace_start_ts; - - // if (ofs_map.find(req.ns) == ofs_map.end()) { - // string output_filepath = ofilepath + ".ns" + to_string(req.ns) + - // ".bin"; ofstream ofs(output_filepath, ios::out | ios::binary); if - // (!ofs.is_open()) { - // cerr << "open " + output_filepath + " failed, check ulimit" << endl; - // abort(); - // } - // ofs_map[req.ns] = std::move(ofs); - // } - // ofs_map[req.ns].write((char *)&(req.ts), sizeof(uint32_t)); - // ofs_map[req.ns].write((char *)&(req.obj_id), sizeof(uint64_t)); - // ofs_map[req.ns].write((char *)&(req.sz), sizeof(uint32_t)); - - buffer_map[req.ns].push_back(req); - if (buffer_map[req.ns].size() >= BUFFER_SIZE) { - if (ofs_map.find(req.ns) == ofs_map.end()) { - string output_filepath = ofilepath + ".ns" + to_string(req.ns) + ".bin"; - ofstream ofs(output_filepath, ios::out | ios::binary); - if (!ofs.is_open()) { - cerr << "open " + output_filepath + " failed, check ulimit" << endl; - abort(); - } - ofs_map[req.ns] = std::move(ofs); - } - - for (auto &req : buffer_map[req.ns]) { - // ofs_map[req.ns].write((char *)&req, 4+8+4); - - ofs_map[req.ns].write((char *)&(req.ts), sizeof(uint32_t)); - ofs_map[req.ns].write((char *)&(req.obj_id), sizeof(uint64_t)); - ofs_map[req.ns].write((char *)&(req.sz), sizeof(uint32_t)); - } - buffer_map[req.ns].clear(); - } - } - - for (auto &t : buffer_map) { - if (ofs_map.find(t.first) != ofs_map.end()) { - for (auto &req : t.second) { - assert(req.ns == t.first); - ofs_map[req.ns].write((char *)&(req.ts), sizeof(uint32_t)); - ofs_map[req.ns].write((char *)&(req.obj_id), sizeof(uint64_t)); - ofs_map[req.ns].write((char *)&(req.sz), sizeof(uint32_t)); - } - } - t.second.clear(); - } - - ifs.close(); - for (auto &kv : ofs_map) { - kv.second.close(); - } - - return 0; -} diff --git a/libCacheSim/bin/dep/misc/flash.cpp b/libCacheSim/bin/dep/misc/flash.cpp deleted file mode 100644 index 96d50f5c..00000000 --- a/libCacheSim/bin/dep/misc/flash.cpp +++ /dev/null @@ -1,127 +0,0 @@ - - -#include "flash.hpp" - -#include - -#include - -#define REPORT_INTERVAL (4 * 3600) -#define MILLION 1000000ul - -void calWriteAmp(reader_t *reader, cache_t *cache) { - /* random seed */ - srand(time(NULL)); - set_rand_seed(rand()); - - request_t *req = new_request(); - read_one_req(reader, req); - uint64_t start_ts = (uint64_t)req->clock_time; - req->clock_time = 0; - - std::unordered_set seen; - - int64_t n_req = 0, n_byte = 0, n_uniq_obj = 0, n_uniq_byte = 0; - int64_t n_miss = 0, n_miss_byte = 0; - while (req->valid) { - n_req++; - n_byte += req->obj_size; - if (seen.find(req->obj_id) == seen.end()) { - seen.insert(req->obj_id); - n_uniq_obj++; - n_uniq_byte += req->obj_size; - } - - if (cache->get(cache, req) == false) { - n_miss += 1; - n_miss_byte += req->obj_size; - } - - read_one_req(reader, req); - req->clock_time -= start_ts; - } - - // printf("%s %32s n_miss %.2lf Mobj (%.4lf, %.4lf), %.2lf MB (%.4lf, %.4lf)", - // basename(reader->trace_path), cache->cache_name, - // n_miss / (double)MILLION, n_miss / (double)n_req, - // n_miss / (double)n_uniq_obj, n_miss_byte / (double)MB, - // n_miss_byte / (double)n_byte, n_miss_byte / (double)n_uniq_byte); - - printf("%s %32s miss ratio %.4lf %.4lf, ", basename(reader->trace_path), - cache->cache_name, n_miss / (double)n_req, - n_miss_byte / (double)n_byte); - - int64_t n_byte_write = n_miss_byte; - if (strcasecmp("fifo", cache->cache_name) == 0) { - ; - } else if (strcasestr(cache->cache_name, "flashProb") != NULL) { - flashProb_params_t *params = (flashProb_params_t *)cache->eviction_params; - - n_byte_write = params->n_byte_admit_to_disk; - - if (strcasestr(params->disk->cache_name, "Clock") != NULL) { - Clock_params_t *clock_params = - (Clock_params_t *)params->disk->eviction_params; - n_byte_write += clock_params->n_byte_rewritten; - } - } else if (strcasestr(cache->cache_name, "clock") != NULL && - strcasestr(cache->cache_name, "qdlp") == NULL) { - Clock_params_t *params = (Clock_params_t *)cache->eviction_params; - - n_byte_write = n_miss_byte + params->n_byte_rewritten; - } else if (strcasestr(cache->cache_name, "FIFO_Reinsertion") != NULL) { - FIFO_Reinsertion_params_t *params = - (FIFO_Reinsertion_params_t *)cache->eviction_params; - - n_byte_write = n_miss_byte + params->n_byte_rewritten; - } else if (strcasestr(cache->cache_name, "qdlpv1") != NULL) { - QDLPv1_params_t *params = (QDLPv1_params_t *)cache->eviction_params; - if (strcasestr(params->main_cache->cache_name, "Clock") != NULL) { - Clock_params_t *clock_params = - (Clock_params_t *)params->main_cache->eviction_params; - n_byte_write = - (params->n_byte_admit_to_main + params->n_byte_move_to_main + - clock_params->n_byte_rewritten); - } else if (strcasestr(params->main_cache->cache_name, "fifo") != NULL) { - // n_obj_write = params->n_obj_admit_to_main + params->n_obj_move_to_main; - n_byte_write = params->n_byte_admit_to_main + params->n_byte_move_to_main; - } else { - ERROR("Unknown main cache type %s\n", params->main_cache->cache_name); - } - } else { - ; - } - - printf("write_amp %.4lf\n", n_byte_write / (double)n_uniq_byte); - - if (strcasestr(cache->cache_name, "qdlpv1") != NULL) { - QDLPv1_params_t *params = (QDLPv1_params_t *)cache->eviction_params; - Clock_params_t *clock_params = - (Clock_params_t *)params->main_cache->eviction_params; - - printf("%.4lf/%.2lf/%.2lf/%.2lf\n", - params->n_byte_admit_to_main / (double)params->n_byte_admit_to_fifo, - params->n_byte_admit_to_main / (double)n_uniq_byte, - params->n_byte_move_to_main / (double)n_uniq_byte, - clock_params->n_byte_rewritten / (double)n_uniq_byte); - } -} - -int main(int argc, char *argv[]) { - struct arguments args; - parse_cmd(argc, argv, &args); - if (args.n_cache_size != 1) { - WARN("only support one cache size\n"); - exit(0); - } - if (args.n_eviction_algo != 1) { - WARN("only support one eviction algorithm\n"); - exit(0); - } - - calWriteAmp(args.reader, args.caches[0]); - - free_arg(&args); - - return 0; -} diff --git a/libCacheSim/bin/dep/misc/flash.hpp b/libCacheSim/bin/dep/misc/flash.hpp deleted file mode 100644 index 456d87a6..00000000 --- a/libCacheSim/bin/dep/misc/flash.hpp +++ /dev/null @@ -1,72 +0,0 @@ - -#include "../../include/libCacheSim/cache.h" -#include "../../include/libCacheSim/reader.h" -#include "../../utils/include/mymath.h" -#include "../../utils/include/mystr.h" -#include "../../utils/include/mysys.h" -#include "internal.h" - - -typedef struct { - cache_t *ram; - cache_t *disk; - - int64_t n_obj_admit_to_ram; - int64_t n_obj_admit_to_disk; - int64_t n_byte_admit_to_ram; - int64_t n_byte_admit_to_disk; - - double ram_size_ratio; - double disk_admit_prob; - int inv_prob; - - char ram_cache_type[32]; - request_t *req_local; -} flashProb_params_t; - -typedef enum { - RETAIN_POLICY_RECENCY = 0, - RETAIN_POLICY_FREQUENCY, - RETAIN_POLICY_BELADY, - RETAIN_NONE -} retain_policy_t; -typedef struct FIFO_Reinsertion_params { - cache_obj_t *q_head; - cache_obj_t *q_tail; - - // points to the eviction position - cache_obj_t *next_to_merge; - // the number of object to examine at each eviction - int n_exam_obj; - // of the n_exam_obj, we keep n_keep_obj and evict the rest - int n_keep_obj; - // used to sort the n_exam_obj objects - struct sort_list_node *metric_list; - // the policy to determine the n_keep_obj objects - retain_policy_t retain_policy; - - int64_t n_obj_rewritten; - int64_t n_byte_rewritten; -} FIFO_Reinsertion_params_t; - -typedef struct { - cache_t *fifo; - cache_t *fifo_ghost; - cache_t *main_cache; - bool hit_on_ghost; - - int64_t n_obj_admit_to_fifo; - int64_t n_obj_admit_to_main; - int64_t n_obj_move_to_main; - int64_t n_obj_rewritten_in_main; - int64_t n_byte_admit_to_fifo; - int64_t n_byte_admit_to_main; - int64_t n_byte_move_to_main; - int64_t n_byte_rewritten_in_main; - - double fifo_size_ratio; - double ghost_size_ratio; - char main_cache_type[32]; - - request_t *req_local; -} QDLPv1_params_t; diff --git a/libCacheSim/bin/dep/oracleTraceGen/CMakeLists.txt b/libCacheSim/bin/dep/oracleTraceGen/CMakeLists.txt deleted file mode 100644 index 01226347..00000000 --- a/libCacheSim/bin/dep/oracleTraceGen/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ - - -add_executable(oracleTraceGen main.cpp) -target_link_libraries(oracleTraceGen oracleTraceGenLib dataStructureLib traceReader ${Boost_LIBRARIES}) diff --git a/libCacheSim/bin/dep/oracleTraceGen/main.cpp b/libCacheSim/bin/dep/oracleTraceGen/main.cpp deleted file mode 100644 index 11382242..00000000 --- a/libCacheSim/bin/dep/oracleTraceGen/main.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// Created by jason on 3/19/21. -// - - -#include "../../oracleTraceGen/oracleReuseTraceGen.h" -#include "../../oracleTraceGen/oracleFreqTraceGen.h" -#include "../utils.h" - -#include "../../traceReader/reader.h" - -using namespace std; -using namespace oracleTraceGen; -namespace po = boost::program_options; - - -typedef struct { - std::string output_trace_type; - - std::string oracle_type; -} oracleTraceGen_arg_t; - -int main(int argc, char *argv[]) { - oracleTraceGen_arg_t oarg; - - po::options_description desc("oracleTraceGen options"); - desc.add_options() - ("output_trace_type", po::value(&(oarg.output_trace_type))->default_value(""), - "output_trace_type [oracleSysTwrNS/oracleAkamai/oracleWiki16u/oracleWiki19u/oracleGeneral]/oracleGeneralOpNS") - ("oracle_type", po::value(&(oarg.oracle_type))->default_value("reuse"), "freq/reuse"); - - cli::cli_arg_t cli_arg = cli::Util::parse_cli_arg(argc, argv, desc); - if (cli_arg.opath.empty()) { - cli_arg.opath = cli_arg.ipath + ".oracle"; - } - - std::cout << "input traces " << cli_arg.ipath << ", type " << cli_arg.trace_type - << ", output " << oarg.output_trace_type << std::endl; - - const char *ipath_cstr = cli_arg.ipath.c_str(); - size_t len = strlen(ipath_cstr); - if (strcasecmp(ipath_cstr + len - 4, ".zst") == 0) { - printf("zstd trace is not supported because of reading from the back\n"); - exit(1); - } - - reader_t *reader = cli::Util::create_reader(&cli_arg); - - if (oarg.oracle_type == "freq") { - /* this is no longer used */ - std::vector time_window{TIME_WINDOW}; - OracleFreqTraceGen trace_gen(reader, cli_arg.opath, time_window, 300); - trace_gen.run(); - } else if (oarg.oracle_type == "reuse") { - if (oarg.output_trace_type == "oracleSysTwrNS") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, true, false); - } else if (oarg.output_trace_type == "oracleCF1") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, true); - } else if (oarg.output_trace_type == "oracleAkamai") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, true); - } else if (oarg.output_trace_type == "oracleWiki16u") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, true); - } else if (oarg.output_trace_type == "oracleWiki19u") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, true); - } else if (oarg.output_trace_type == "oracleGeneral") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, true); - } else if (oarg.output_trace_type == "oracleGeneralOpNS") { - OracleReuseTraceGen trace_gen(reader, cli_arg.opath); - trace_gen.run(false, false, false); - } else { - std::cerr << "unknown output type " << oarg.output_trace_type << std::endl; - abort(); - } - } else { - std::cerr << "unknown oracle " << oarg.oracle_type << std::endl; - abort(); - } -} - - - diff --git a/libCacheSim/bin/dep/oracleTraceGen/oracleReuseTraceGen.h b/libCacheSim/bin/dep/oracleTraceGen/oracleReuseTraceGen.h deleted file mode 100644 index 2a4e8413..00000000 --- a/libCacheSim/bin/dep/oracleTraceGen/oracleReuseTraceGen.h +++ /dev/null @@ -1,631 +0,0 @@ -#pragma once - -#include "../traceReader/reader.h" -#include "enum.h" -#include "utilsPrint.h" -#include "../utils/include/utils.h" -#include "../lib/robin_hood.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace oracleTraceGen { - -/* this considers update and delete, which resets next_access_vtime_ */ -struct reqEntryReuseRealSystemTwrNS { - // IQHIHHIQ - uint32_t real_time_; - uint64_t obj_id_; - uint16_t key_size_; - uint32_t val_size_; - uint16_t op_; - uint16_t ns; /* namespace */ - int32_t ttl_; - int64_t next_access_vtime_; - - int get_obj_size() { - return key_size_ + val_size_; - } - void set_obj_size(int obj_size) { - ; - } - bool is_size_zero() { - return val_size_ == 0 && (op_ == OP_GET || op_ == OP_GETS); - } - bool is_op_write_or_delete() { - /* add is not counted as write, see enum.h for op definition */ - return op_ == 3 || (op_ >= 5 && op_ <= 9); - } - int32_t get_ttl() { - return ttl_; - } - void set_ttl(int32_t ttl) { - ttl_ = ttl; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->key_size_ = req->key_size; - this->val_size_ = req->val_size; - this->ns = req->obj_namespace; - this->op_ = req->op; - this->ttl_ = req->ttl; - } -}__attribute__((packed)); - -// ts, obj, sz, ttl, age, hostname, content (h), extension (h), n_level, n_param, method, colo -struct reqEntryReuseCF1 { - uint32_t real_time_; - uint64_t obj_id_; - uint64_t obj_size_; - uint32_t ttl_; - uint32_t age_; - uint32_t hostname_; - int64_t next_access_vtime_; - uint16_t content_type_; - uint16_t extension_; - uint16_t n_level_; - uint8_t n_param_; - uint8_t method_; - uint8_t colo_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_size_zero() { - return obj_size_ == 0; - } - bool is_op_write_or_delete() { - return false; - } - int32_t get_ttl() { - return ttl_; - } - void set_ttl(int32_t ttl) { - ttl_ = ttl; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - this->ttl_ = req->ttl; - this->age_ = req->age; - this->hostname_ = req->hostname; - this->extension_ = req->extension; - this->n_level_ = req->n_level; - this->n_param_ = req->n_param; - this->method_ = req->method; - this->colo_ = req->colo; - - this->content_type_ = req->content_type; - } -}__attribute__((packed)); - -struct reqEntryReuseAkamai { - uint32_t real_time_; - uint64_t obj_id_; - uint32_t obj_size_; - int16_t customer_id_; - int16_t bucket_id_; - int16_t content_type_; - int64_t next_access_vtime_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_size_zero() { - return obj_size_ == 0; - } - bool is_op_write_or_delete() { - return false; - } - int32_t get_ttl() { - return 0; - } - void set_ttl(int32_t ttl) { - ; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - this->customer_id_ = req->tenant_id; - this->bucket_id_ = req->bucket_id; - this->content_type_ = req->content_type; - } -}__attribute__((packed)); - -struct reqEntryReuseWiki16u { - uint64_t obj_id_; - uint32_t obj_size_; - int16_t content_type_; - int64_t next_access_vtime_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_op_write_or_delete() { - return false; - } - bool is_size_zero() { - return obj_size_ == 0; - } - int32_t get_ttl() { - return 0; - } - void set_ttl(int32_t ttl_) { - ; - } - uint32_t get_real_time() { - return 0; - } - void init(request_t *req) { - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - this->content_type_ = req->content_type; - } -}__attribute__((packed)); - -struct reqEntryReuseWiki19u { - uint32_t real_time_; - uint64_t obj_id_; - uint32_t obj_size_; - int16_t content_type_; - int64_t next_access_vtime_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_size_zero() { - return obj_size_ == 0; - } - bool is_op_write_or_delete() { - return false; - } - int32_t get_ttl() { - return 0; - } - void set_ttl(int32_t ttl_) { - ; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - this->content_type_ = req->content_type; - } -}__attribute__((packed)); - -struct reqEntryReuseGeneral { - uint32_t real_time_; - uint64_t obj_id_; - uint32_t obj_size_; - int64_t next_access_vtime_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_size_zero() { - return obj_size_ == 0; - } - bool is_op_write_or_delete() { - return false; - } - int32_t get_ttl() { - return 0; - } - void set_ttl(int32_t ttl) { - ; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - if (req->obj_size >= UINT32_MAX) { - printf("obj_size %ld is too large, set to UINT32_MAX\n", req->obj_size); - this->obj_size_ = UINT32_MAX; - } - } -}__attribute__((packed)); - -/* this considers update and delete, which resets next_access_vtime_ */ -struct reqEntryReuseGeneralOpNS { - // IqIbhq - uint32_t real_time_; - uint64_t obj_id_; - uint32_t obj_size_; - uint8_t op_; - uint16_t ns; /* namespace */ - int64_t next_access_vtime_; - - int get_obj_size() { - return obj_size_; - } - void set_obj_size(int obj_size) { - this->obj_size_ = obj_size; - } - bool is_size_zero() { - return obj_size_ == 0; - } - bool is_op_write_or_delete() { - /* add is not counted as write, see enum.h for op definition */ - return op_ == 3; - } - int32_t get_ttl() { - return 0; - } - void set_ttl(int32_t ttl) { - ; - } - uint32_t get_real_time() { - return real_time_; - } - void init(request_t *req) { - this->real_time_ = req->real_time; - this->obj_id_ = req->obj_id; - this->obj_size_ = req->obj_size; - this->ns = req->obj_namespace; - this->op_ = req->op; - } -}__attribute__((packed)); - - -template -class OracleReuseTraceGen { - public: - OracleReuseTraceGen(reader_t *reader, std::string ofile_path) : - reader_(reader), - ofile_path_(std::move(ofile_path)) { - temp_ofile_path_ = ofile_path_ + ".reverse"; - ofile_temp_.open(temp_ofile_path_, std::ios::out | std::ios::binary | std::ios::trunc); - ofile_.open(ofile_path_, std::ios::out |std:: ios::binary | std::ios::trunc); - if (!ofile_) { - std::cerr << "cannot open output file " << ofile_path_ << std::endl; - } - } - - ~OracleReuseTraceGen() { - if (std::remove(temp_ofile_path_.c_str()) != 0) { - perror("unable to remove the temporary file\n"); - } - }; - OracleReuseTraceGen(const OracleReuseTraceGen &seg_orcale) = delete; - OracleReuseTraceGen(const OracleReuseTraceGen &&seg_orcale) = delete; - OracleReuseTraceGen &operator=(const OracleReuseTraceGen &o) = delete; - OracleReuseTraceGen &operator=(const OracleReuseTraceGen &&o) = delete; - - - struct last_access_info { - int64_t vtime; - int32_t ttl; - }__attribute__((packed)); - void run(bool output_txt, bool backfill_ttl, bool remove_size_change) { - request_t *req = new_request(); - auto *last_access= new robin_hood::unordered_flat_map; - - /* n_req_written counts the number of requests written to file, - * n_req_total is the total number of requests in the trace - * n_req_curr is the number of requests scanned, this is different from n_req_written when - * we remove miss caused fills in twr traces */ - uint64_t n_req_written = 0, n_req_curr = 0, n_req_total = get_num_of_req(reader_); -#ifndef __APPLE__ - last_access->reserve(1e8); -#endif - - reader_->ignore_size_zero_req = FALSE; - /* we remove cache miss caused on-demand fill for twr trace */ - boost::dynamic_bitset<> *is_cache_miss_bitset = nullptr; - int64_t n_miss_fill = 0; - if (typeid(T) == typeid(struct reqEntryReuseRealSystemTwrNS)) { - is_cache_miss_bitset = std::move(gen_miss_fill_vec(&n_miss_fill)); - } - - int status; - uint64_t unique_bytes = 0, total_bytes = 0; - int64_t n_ttl_missing = 0, n_size_zero = 0; - utilsPrint::print_time(); - std::cout << reader_->trace_path << ": " << (double) n_req_total / 1000000.0 << "M requests, " << n_miss_fill << " miss fill" << std::endl; - - /* we read reversely and ignore_size_zero_req will read forward */ - reader_set_read_pos(reader_, 1.0); - go_back_one_line(reader_); - read_one_req(reader_, req); - - int64_t start_ts = req->real_time; - - T *re = new T; - re->init(req); - int32_t last_ttl; - int64_t last_access_vtime; - - while (true) { - if (typeid(T) == typeid(struct reqEntryReuseRealSystemTwrNS) && (*is_cache_miss_bitset)[n_req_total - n_req_curr - 1]) { - /* this is on-demand fill, because we remove the get miss (they have zero size), - * we change the on-demand fill to GET request, which will cause an on-demand fill if it is cache miss */ - req->op = OP_GET; - } - - last_ttl = 0; - auto last_access_it = last_access->find(re->obj_id_); - - if (backfill_ttl) { - if (re->get_ttl() != 0) { - last_ttl = re->get_ttl(); - } else { - if (last_access_it != last_access->end()) { - re->set_ttl(last_access_it->second.ttl); - last_ttl = last_access_it->second.ttl; - } else { - n_ttl_missing += 1; - } - } - } - - if (last_access_it == last_access->end()) { - re->next_access_vtime_ = -1; - unique_bytes += req->obj_size; - } else { - re->next_access_vtime_ = last_access_it->second.vtime; - } - - total_bytes += req->obj_size; - - last_access_vtime = (int64_t) n_req_written; - - if (typeid(T) == typeid(struct reqEntryReuseRealSystemTwrNS) && re->is_op_write_or_delete()) { - /* if this request is write or delete */ - last_access_vtime = -1; - } - - - (*last_access)[req->obj_id] = {.vtime = last_access_vtime, .ttl = last_ttl}; - - ofile_temp_.write(reinterpret_cast(re), sizeof(T)); - n_req_written += 1; - - if (n_req_curr % 100000000 == 0 && n_req_curr != 0) { - utilsPrint::print_time(); - std::cout << std::fixed << std::setprecision(2) << (double) n_req_curr / 1000000 << " MReq, ts " << -req->real_time + start_ts - << ", working set/req GB " << (double) unique_bytes / GiB << "/" << (double) total_bytes / GiB << std::endl; - } - - status = read_one_req_above(reader_, req); - re->init(req); - n_req_curr += 1; - - while (re->is_size_zero() && status == 0) { - /* we skip zero-sized request, which is either something does not exist - * or a cache miss, and the latter will trigger an on-demand fill */ - n_size_zero += 1; - status = read_one_req_above(reader_, req); - re->init(req); - n_req_curr += 1; - } - - if (status != 0) - break; - } - - utils::my_assert(n_req_curr == n_req_total, "n_req_curr != n_req_total"); - - delete re; - free_request(req); - - ofile_temp_.close(); - delete last_access; - delete is_cache_miss_bitset; - - utilsPrint::print_time(); - std::cout << ofile_path_ << " "; - std::cout << n_size_zero << "(" << (double) n_size_zero/(double) n_req_total * 100 << "%) size zero requests"; - if (backfill_ttl) { - std::cout << ", missing ttl " << n_ttl_missing << "/" << n_req_written << " (" << (double) n_ttl_missing / (double) n_req_written << ")\n"; - } else { - std::cout << std::endl; - } - - _reverse_file(n_req_written, output_txt, remove_size_change); - } - - private: - reader_t *reader_; - std::string ofile_path_; - std::string temp_ofile_path_; - std::ofstream ofile_; - std::ofstream ofile_txt_; - /* TODO: change to tempfile */ - std::ofstream ofile_temp_; - - boost::dynamic_bitset<> *gen_miss_fill_vec(int64_t *n_fill) { - /* for twr traces, find whether a cache write is an on-demand fill */ - uint64_t n_req = get_num_of_req(reader_); - boost::dynamic_bitset<> *is_cache_miss_bitset = new boost::dynamic_bitset<>(n_req); - robin_hood::unordered_flat_map last_miss_ts; - - request_t *req = new_request(); - - read_one_req(reader_, req); - int64_t n_req_curr = 1; - while (req->valid) { -// printf("%ld time %ld obj_id %lu obj_size %d + %u op %d namespace %d ttl %d next %ld xxxxxxxx \n", (long) n_req_curr, -// (long) req->real_time, (unsigned long) req->obj_id, (int) req->key_size, (unsigned) req->val_size, (int) req->op, -// (int) req->obj_namespace, (int) req->ttl, (long) req->next_access_vtime); - - if (req->val_size == 0 && (req->op == OP_GET || req->op == OP_GETS)) { - last_miss_ts[req->obj_id] = req->real_time; - } - - if (req->op == OP_SET || req->op == OP_ADD || req->op == OP_CAS || req->op == OP_REPLACE) { - auto it = last_miss_ts.find(req->obj_id); - if (it != last_miss_ts.end() && req->real_time - it->second <= 1) { - (*is_cache_miss_bitset)[n_req_curr] = 1; - *n_fill += 1; - } - } - - read_one_req(reader_, req); - n_req_curr += 1; - } - utils::my_assert(n_req_curr -1 == n_req, "n_req_curr - 1 != n_req"); - - free_request(req); - reset_reader(reader_); - - return is_cache_miss_bitset; - } - - void *_setup_mmap(std::string &file_path, size_t *size) { - int fd; - struct stat st{}; - void *mapped_file; - - // set up mmap region - if ((fd = open(file_path.c_str(), O_RDONLY)) < 0) { - ERROR("Unable to open '%s', %s\n", file_path.c_str(), strerror(errno)); - exit(1); - } - - if ((fstat(fd, &st)) < 0) { - close(fd); - ERROR("Unable to fstat '%s', %s\n", file_path.c_str(), strerror(errno)); - exit(1); - } - - *size = st.st_size; - mapped_file = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - if ((mapped_file) == MAP_FAILED) { - close(fd); - mapped_file = nullptr; - ERROR("Unable to allocate %llu bytes of memory, %s\n", - (unsigned long long) st.st_size, strerror(errno)); - abort(); - } - -#ifdef MADV_HUGEPAGE - int mstatus = madvise(mapped_file, st.st_size, MADV_HUGEPAGE | MADV_SEQUENTIAL); - if (mstatus != 0) { - ERROR("cannot turn on hugepage %s\n", strerror(errno)); - } -#endif - - - close(fd); - return mapped_file; - } - - void _reverse_file(uint64_t n_total_req, bool output_txt, bool remove_size_change) { - utilsPrint::print_time(); - uint64_t n_req = 0; - size_t file_size; - char *mapped_file = reinterpret_cast(_setup_mmap(temp_ofile_path_, &file_size)); - size_t pos = file_size; - - if (output_txt) - ofile_txt_.open(ofile_path_ + ".txt", std::ios::out | std::ios::trunc); - - /* we remove object size change because some of the systems do not allow object size change */ -// std::unordered_map last_obj_size; - robin_hood::unordered_flat_map last_obj_size; - -#ifndef __APPLE__ - last_obj_size.reserve(1e8); -#endif - - T re; - size_t req_entry_size = sizeof(re); - std::cout << "reverse output " << n_total_req << " requests, " << "entry size " << req_entry_size << std::endl; - - while (pos >= req_entry_size) { - pos -= req_entry_size; - memcpy(&re, mapped_file + pos, req_entry_size); - if (re.next_access_vtime_ != -1) { - /* re.next_access_vtime_ is the vtime start from the end */ - re.next_access_vtime_ = n_total_req - 1 - re.next_access_vtime_; - } - - int32_t obj_size = re.get_obj_size(); - if (remove_size_change) { - auto it = last_obj_size.find(re.obj_id_); - if (it != last_obj_size.end()) { - obj_size = it->second; - re.set_obj_size(obj_size); - } else { - last_obj_size[re.obj_id_] = obj_size; - } - } - - - ofile_.write(reinterpret_cast(&re), req_entry_size); - if (output_txt) { - if (typeid(T) == typeid(struct reqEntryReuseRealSystemTwrNS)) { - auto re2 = reinterpret_cast(&re); - ofile_txt_ << re.get_real_time() << " " << re.obj_id_ << " " << re2->op_ << " " << re.next_access_vtime_ << "\n"; - } else { - ofile_txt_ << re.get_real_time() << " " << re.obj_id_ << " " << re.next_access_vtime_ << "\n"; - } - } - n_req += 1; - - if (n_req % 100000000 == 0) { - utilsPrint::print_time(); - std::cout << "write " << std::setprecision(4) << (double) n_req / 1000000 << " MReq" << std::endl; - } - } - - munmap(mapped_file, file_size); - ofile_.close(); - if (output_txt) - ofile_txt_.close(); - } -}; -} - - - - - - - diff --git a/libCacheSim/bin/dep/traceOneHit/CMakeLists.txt b/libCacheSim/bin/dep/traceOneHit/CMakeLists.txt deleted file mode 100644 index 2e026456..00000000 --- a/libCacheSim/bin/dep/traceOneHit/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ - -add_library(traceOneHitLib oneHit.cpp) - - -add_executable(traceOneHit main.c cli_parser.c ../cli_reader_utils.c) -target_link_libraries(traceOneHit traceOneHitLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) -install(TARGETS traceOneHit RUNTIME DESTINATION bin) diff --git a/libCacheSim/bin/dep/traceOneHit/cli_parser.c b/libCacheSim/bin/dep/traceOneHit/cli_parser.c deleted file mode 100644 index 8be15bd5..00000000 --- a/libCacheSim/bin/dep/traceOneHit/cli_parser.c +++ /dev/null @@ -1,164 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include - -#include "../../include/libCacheSim/const.h" -#include "../../utils/include/mystr.h" -#include "../../utils/include/mysys.h" -#include "../cli_reader_utils.h" -#include "internal.h" - -#ifdef __cplusplus -extern "C" { -#endif - -const char *argp_program_version = "traceAnalysis 0.0.1"; -const char *argp_program_bug_address = - "https://groups.google.com/g/libcachesim/"; - -enum argp_option_short { - OPTION_TRACE_TYPE_PARAMS = 't', - OPTION_OUTPUT_PATH = 'o', - OPTION_NUM_REQ = 'n', - OPTION_VERBOSE = 'v', -}; - -/* - OPTIONS. Field 1 in ARGP. - Order of fields: {NAME, KEY, ARG, FLAGS, DOC}. -*/ -static struct argp_option options[] = { - {"trace-type-params", OPTION_TRACE_TYPE_PARAMS, - "\"obj-id-col=1;delimiter=,\"", 0, - "Parameters used for csv trace, e.g., \"obj-id-col=1;delimiter=,\"", 2}, - {"num-req", OPTION_NUM_REQ, "-1", 0, - "Num of requests to process, default -1 means all requests in the trace"}, - - {"output", OPTION_OUTPUT_PATH, "output", 0, "Output path", 5}, - {"verbose", OPTION_VERBOSE, "1", 0, "Produce verbose output"}, - - {0}}; - -/* - PARSER. Field 2 in ARGP. - Order of parameters: KEY, ARG, STATE. -*/ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - struct arguments *arguments = state->input; - - switch (key) { - case OPTION_TRACE_TYPE_PARAMS: - arguments->trace_type_params = arg; - break; - case OPTION_OUTPUT_PATH: - strncpy(arguments->ofilepath, arg, OFILEPATH_LEN - 1); - arguments->ofilepath[OFILEPATH_LEN - 1] = '\0'; - break; - case OPTION_NUM_REQ: - arguments->n_req = atoi(arg); - break; - case OPTION_VERBOSE: - arguments->verbose = is_true(arg) ? true : false; - break; - case ARGP_KEY_ARG: - if (state->arg_num >= N_ARGS) { - printf("found too many arguments, current %s\n", arg); - argp_usage(state); - exit(1); - } - arguments->args[state->arg_num] = arg; - break; - case ARGP_KEY_END: - if (state->arg_num < N_ARGS) { - printf("not enough arguments found\n"); - argp_usage(state); - exit(1); - } - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -/* - ARGS_DOC. Field 3 in ARGP. - A description of the non-option command-line arguments - that we accept. -*/ -static char args_doc[] = "trace_path trace_type task"; - -/* Program documentation. */ -static char doc[] = - "example: ./traceAnalysis /trace/path csv calOneHit\n\n" - "trace_type: txt/csv/twr/vscsi/oracleGeneralBin and more\n" - "if using csv trace, considering specifying -t obj-id-is-num=true\n\n" - "task: calOneHit\n\n"; - -/** - * @brief initialize the arguments - * - * @param args - */ -static void init_arg(struct arguments *args) { - memset(args, 0, sizeof(struct arguments)); - - args->trace_path = NULL; - args->trace_type_params = NULL; - args->verbose = true; - memset(args->ofilepath, 0, OFILEPATH_LEN); - args->n_req = -1; -} - -/** - * @brief parse the command line arguments - * - * @param argc - * @param argv - */ -void parse_cmd(int argc, char *argv[], struct arguments *args) { - init_arg(args); - - static struct argp argp = {.options = options, - .parser = parse_opt, - .args_doc = args_doc, - .doc = doc, - .children = NULL, - .help_filter = NULL, - .argp_domain = NULL}; - - argp_parse(&argp, argc, argv, 0, 0, args); - - args->trace_path = args->args[0]; - const char *trace_type_str = args->args[1]; - const char *task = args->args[2]; - strncpy(args->task, task, TASK_STR_LEN - 1); - args->task[TASK_STR_LEN - 1] = '\0'; - - /* convert trace type string to enum */ - args->trace_type = trace_type_str_to_enum(trace_type_str, args->trace_path); - - reader_init_param_t reader_init_params; - memset(&reader_init_params, 0, sizeof(reader_init_params)); - reader_init_params.ignore_size_zero_req = true; - reader_init_params.obj_id_is_num = true; - reader_init_params.cap_at_n_req = args->n_req; - reader_init_params.sampler = NULL; - - parse_reader_params(args->trace_type_params, &reader_init_params); - - if ((args->trace_type == CSV_TRACE || args->trace_type == PLAIN_TXT_TRACE) && - reader_init_params.obj_size_field == -1) { - reader_init_params.ignore_obj_size = true; - } - - args->reader = - setup_reader(args->trace_path, args->trace_type, &reader_init_params); -} - -void free_arg(struct arguments *args) { close_reader(args->reader); } - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/libCacheSim/bin/dep/traceOneHit/internal.h b/libCacheSim/bin/dep/traceOneHit/internal.h deleted file mode 100644 index e9f81f9e..00000000 --- a/libCacheSim/bin/dep/traceOneHit/internal.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include - -#include "../../include/libCacheSim/enum.h" -#include "../../include/libCacheSim/reader.h" - -#define N_ARGS 3 -#define OFILEPATH_LEN 128 -#define TASK_STR_LEN 64 - -#ifdef __cplusplus -extern "C" { -#endif - -/* This structure is used to communicate with parse_opt. */ -struct arguments { - /* argument from the user */ - char *args[N_ARGS]; - char *trace_path; - char ofilepath[OFILEPATH_LEN]; - trace_type_e trace_type; - char *trace_type_params; - char task[TASK_STR_LEN]; - int64_t n_req; /* number of requests to process */ - bool verbose; - - /* arguments generated */ - reader_t *reader; -}; - -void parse_cmd(int argc, char *argv[], struct arguments *args); - -void cal_one_hit(reader_t *reader, char *ofilepath); - - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/bin/dep/traceOneHit/main.c b/libCacheSim/bin/dep/traceOneHit/main.c deleted file mode 100644 index aeb564c2..00000000 --- a/libCacheSim/bin/dep/traceOneHit/main.c +++ /dev/null @@ -1,24 +0,0 @@ - -#include - -#include "internal.h" - -int main(int argc, char *argv[]) { - struct arguments args; - parse_cmd(argc, argv, &args); - - if (strcasecmp(args.task, "calOneHit") == 0) { - if (args.ofilepath[0] == '\0') { - snprintf(args.ofilepath, OFILEPATH_LEN, "%s.oneHit", - basename(args.trace_path)); - } - - cal_one_hit(args.reader, args.ofilepath); - } else { - fprintf(stderr, "Unknown task: %s\n", args.task); - } - - close_reader(args.reader); - - return 0; -} diff --git a/libCacheSim/bin/dep/traceOneHit/oneHit.cpp b/libCacheSim/bin/dep/traceOneHit/oneHit.cpp deleted file mode 100644 index 52a48c2d..00000000 --- a/libCacheSim/bin/dep/traceOneHit/oneHit.cpp +++ /dev/null @@ -1,66 +0,0 @@ - - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "../../include/libCacheSim/const.h" -#include "../../utils/include/mystr.h" -#include "../../utils/include/mysys.h" -#include "../cli_reader_utils.h" -#include "internal.h" - -using namespace std; - -void cal_one_hit(reader_t *reader, char *ofilepath) { - unordered_map freq_map; - vector one_hit_ratio_vec; - - request_t *req = new_request(); - - read_one_req(reader, req); - size_t last_size = 0; - uint64_t n_one_hit = 0; - - while (req->valid) { - if (freq_map.find(req->obj_id) == freq_map.end()) { - freq_map[req->obj_id] = 1; - n_one_hit++; - } else { - freq_map[req->obj_id] += 1; - if (freq_map[req->obj_id] == 2) { - n_one_hit--; - } - } - - if (freq_map.size() % 100 == 0 && freq_map.size() != last_size) { - one_hit_ratio_vec.push_back((double)n_one_hit / freq_map.size()); - last_size = freq_map.size(); - - if (one_hit_ratio_vec.size() != freq_map.size() / 100) { - ERROR("Error: one_hit_ratio_vec.size() != freq_map.size() / 100\n"); - } - } - - read_one_req(reader, req); - } - - ofstream ofile(ofilepath, ios::out); - ofile << reader->trace_path << ":"; - for (auto ratio : one_hit_ratio_vec) { - ofile << ratio << ","; - } - ofile << endl; - - ofile.close(); - - printf("%s in total %ld elem %ld objects %.4lf %.4lf\n", reader->trace_path, - one_hit_ratio_vec.size(), freq_map.size(), one_hit_ratio_vec[0], - one_hit_ratio_vec[one_hit_ratio_vec.size() - 1]); -} \ No newline at end of file diff --git a/libCacheSim/bin/distUtil/CMakeLists.txt b/libCacheSim/bin/distUtil/CMakeLists.txt index 9ac7f89b..7effc0dc 100644 --- a/libCacheSim/bin/distUtil/CMakeLists.txt +++ b/libCacheSim/bin/distUtil/CMakeLists.txt @@ -1,4 +1,12 @@ +# ============================================================================== +# DistUtil +# ============================================================================== -add_executable(distUtil main.c cli.c ../cli_reader_utils.c) -target_link_libraries(distUtil ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) -install(TARGETS cachesim RUNTIME DESTINATION bin) +set(distUtil_sources_c + main.c + cli.c + ../cli_reader_utils.c +) +add_executable(distUtil ${distUtil_sources_c}) +target_link_libraries(distUtil ${all_modules} ${dependency_libs}) +install(TARGETS distUtil RUNTIME DESTINATION bin) diff --git a/libCacheSim/bin/mrcProfiler/CMakeLists.txt b/libCacheSim/bin/mrcProfiler/CMakeLists.txt index b609613a..912bd990 100644 --- a/libCacheSim/bin/mrcProfiler/CMakeLists.txt +++ b/libCacheSim/bin/mrcProfiler/CMakeLists.txt @@ -1,10 +1,22 @@ +# ============================================================================== +# MRC Profiler +# ============================================================================== -add_executable(mrcProfiler main.cpp cli_parser.cpp ../cli_reader_utils.c) -target_link_libraries(mrcProfiler mrcProfilerLib m zstd dl pthread - -Wl,--whole-archive - libCacheSim - -Wl,--no-whole-archive - ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils ) +set(mrcProfiler_sources_cpp + main.cpp + cli_parser.cpp + ../cli_reader_utils.c +) +add_executable(mrcProfiler ${mrcProfiler_sources_cpp}) + +# Set target-specific compiler flags for mixed C/C++ sources +target_compile_options(mrcProfiler PRIVATE + $<$:${LIBCACHESIM_C_FLAGS}> + $<$:${LIBCACHESIM_CXX_FLAGS}> + ${tcmalloc_flags} +) + +target_link_libraries(mrcProfiler mrcProfiler_lib ${all_modules} ${dependency_libs} m dl) target_link_options(mrcProfiler PRIVATE "-Wl,--export-dynamic") install(TARGETS mrcProfiler RUNTIME DESTINATION bin) diff --git a/libCacheSim/bin/priv/lesscache/CMakeLists.txt b/libCacheSim/bin/priv/lesscache/CMakeLists.txt index 5af09479..319569c3 100644 --- a/libCacheSim/bin/priv/lesscache/CMakeLists.txt +++ b/libCacheSim/bin/priv/lesscache/CMakeLists.txt @@ -1,4 +1,10 @@ +# ============================================================================== +# LessCache +# ============================================================================== +set(lesscache_sources_c + main.c +) -add_executable(lesscache main.c) -target_link_libraries(lesscache ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) +add_executable(lesscache ${lesscache_sources_c}) +target_link_libraries(lesscache ${all_modules} ${dependency_libs}) diff --git a/libCacheSim/bin/traceAnalyzer/CMakeLists.txt b/libCacheSim/bin/traceAnalyzer/CMakeLists.txt index 42b3bf68..36e6d22a 100644 --- a/libCacheSim/bin/traceAnalyzer/CMakeLists.txt +++ b/libCacheSim/bin/traceAnalyzer/CMakeLists.txt @@ -1,6 +1,14 @@ +# ============================================================================== +# Trace Analyzer +# ============================================================================== -add_executable(traceAnalyzer main.cpp cli_parser.cpp ../cli_reader_utils.c) -target_link_libraries(traceAnalyzer traceAnalyzerLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils) +set(traceAnalyzer_sources_cpp + main.cpp + cli_parser.cpp + ../cli_reader_utils.c +) +add_executable(traceAnalyzer ${traceAnalyzer_sources_cpp}) +target_link_libraries(traceAnalyzer ${all_modules} ${dependency_libs}) install(TARGETS traceAnalyzer RUNTIME DESTINATION bin) set_target_properties(traceAnalyzer diff --git a/libCacheSim/bin/traceUtils/CMakeLists.txt b/libCacheSim/bin/traceUtils/CMakeLists.txt index 508cfcdf..c67addc4 100644 --- a/libCacheSim/bin/traceUtils/CMakeLists.txt +++ b/libCacheSim/bin/traceUtils/CMakeLists.txt @@ -1,10 +1,12 @@ - +# ============================================================================== +# Trace Utils +# ============================================================================== add_library(cliReaderLib ../cli_reader_utils.c) add_executable(tracePrint tracePrintMain.cpp cli_parser.cpp) -target_link_libraries(tracePrint cliReaderLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(tracePrint cliReaderLib ${all_modules} ${dependency_libs}) set_target_properties(tracePrint PROPERTIES CXX_STANDARD 17 @@ -14,7 +16,7 @@ set_target_properties(tracePrint add_executable(traceConv traceConvMain.cpp traceConvOracleGeneral.cpp traceConvLCS.cpp cli_parser.cpp utils.cpp) -target_link_libraries(traceConv cliReaderLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(traceConv cliReaderLib ${all_modules} ${dependency_libs}) set_target_properties(traceConv PROPERTIES CXX_STANDARD 17 @@ -23,7 +25,7 @@ set_target_properties(traceConv ) add_executable(traceFilter traceFilterMain.cpp cli_parser.cpp) -target_link_libraries(traceFilter cliReaderLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(traceFilter cliReaderLib ${all_modules} ${dependency_libs}) set_target_properties(traceFilter PROPERTIES CXX_STANDARD 17 diff --git a/libCacheSim/cache/CMakeLists.txt b/libCacheSim/cache/CMakeLists.txt index a33e7fa7..b700a153 100644 --- a/libCacheSim/cache/CMakeLists.txt +++ b/libCacheSim/cache/CMakeLists.txt @@ -1,14 +1,190 @@ +# ============================================================================== +# Cache Library +# ============================================================================== -include_directories(admission) -include_directories(eviction) -include_directories(prefetch) +# ============================================================================== +# Admission +# ============================================================================== +# C admission algorithms +set(admission_sources_c + admission/bloomfilter.c + admission/prob.c + admission/size.c + admission/sizeProbabilistic.c +) -add_subdirectory(admission) -add_subdirectory(eviction) -add_subdirectory(prefetch) +# C++ admission algorithms +set(admission_sources_cpp + admission/adaptsize/adaptsize.cpp + admission/adaptsize/adaptsize_interface.cpp +) -add_library(cachelib cache.c) -target_link_libraries(cachelib dataStructure) +# ============================================================================== +# Eviction +# ============================================================================== + +# C eviction algorithms +set(eviction_sources_c + eviction/ARC.c + eviction/ARCv0.c # an inefficient version but easier to understand + eviction/Belady.c + eviction/BeladySize.c + eviction/CAR.c + eviction/Cacheus.c + eviction/Clock.c + eviction/ClockPro.c + eviction/FIFO.c + eviction/Hyperbolic.c + eviction/LeCaR.c + eviction/LeCaRv0.c # an inefficient version but easier to understand + eviction/LFU.c + eviction/LFUDA.c + eviction/LIRS.c + eviction/LRU.c + eviction/LRUProb.c # a probabilistic version of LRU + eviction/LRUv0.c # an inefficient version but easier to understand + eviction/MRU.c + eviction/nop.c + eviction/QDLP.c + eviction/Random.c + eviction/RandomLRU.c + eviction/RandomTwo.c + eviction/S3FIFO.c + eviction/Sieve.c + eviction/Size.c + eviction/SLRU.c + eviction/SLRUv0.c # an inefficient version but easier to understand + eviction/TwoQ.c + eviction/WTinyLFU.c + + # these are sub-algorithms used by other algorithms + eviction/CR_LFU.c + eviction/SR_LRU.c + + # these are algorithms used for comparison + eviction/FIFO_Merge.c + eviction/FIFO_Reinsertion.c + eviction/S3FIFOv0.c + eviction/S3FIFOd.c + eviction/other/flashProb.c + eviction/other/S3LRU.c +) + +# C++ eviction algorithms +set(eviction_sources_cpp + eviction/cpp/LFU.cpp + eviction/cpp/GDSF.cpp + eviction/LHD/lhd.cpp + eviction/LHD/LHD_Interface.cpp + ) + +# LRB +if (ENABLE_LRB) + set (eviction_sources_cpp ${eviction_sources_cpp} + eviction/LRB/lrb.cpp + eviction/LRB/LRB_Interface.cpp + ) +endif() + +# 3L Cache +if (ENABLE_3L_CACHE) + set (eviction_sources_cpp ${eviction_sources_cpp} + eviction/3LCache/ThreeLCache_Interface.cpp + eviction/3LCache/ThreeLCache.cpp + ) +endif() + +# GLCache +if (ENABLE_GLCACHE) + set(eviction_sources_c + ${eviction_sources_c} + eviction/GLCache/bucket.c + eviction/GLCache/dataPrep.c + eviction/GLCache/eviction.c + eviction/GLCache/GLCache.c + eviction/GLCache/inference.c + eviction/GLCache/init.c + eviction/GLCache/segment.c + eviction/GLCache/segSel.c + eviction/GLCache/train.c + ) +endif() + + +# ============================================================================== +# Prefetch +# ============================================================================== + +# C prefetch algorithms +set(prefetch_sources_c + prefetch/Mithril.c + prefetch/OBL.c + prefetch/PG.c +) + +# ============================================================================== +# Cache +# ============================================================================== +# Main cache library +set(cache_sources_c + ${admission_sources_c} + ${eviction_sources_c} + ${prefetch_sources_c} + cache.c + plugin.c +) + +set(cache_sources_cpp + ${admission_sources_cpp} + ${eviction_sources_cpp} +) + +set(cache_sources + ${cache_sources_c} + ${cache_sources_cpp} +) + +# Also set for parent scope if needed +set(cache_sources_c ${cache_sources_c} PARENT_SCOPE) +set(cache_sources_cpp ${cache_sources_cpp} PARENT_SCOPE) +set(cache_sources ${cache_sources} PARENT_SCOPE) + +message(STATUS "cache_sources_c = ${cache_sources_c}") +message(STATUS "cache_sources_cpp = ${cache_sources_cpp}") + +add_library(cache_lib_c ${cache_sources_c}) +add_library(cache_lib_cpp ${cache_sources_cpp}) + +# Set target-specific compiler flags +target_compile_options(cache_lib_c PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} +) + +target_compile_options(cache_lib_cpp PRIVATE + ${LIBCACHESIM_CXX_FLAGS} + ${tcmalloc_flags} +) + +target_include_directories(cache_lib_c PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +target_include_directories(cache_lib_cpp PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +target_link_libraries(cache_lib_c dataStructure_lib utils_lib ${dependency_libs}) +target_link_libraries(cache_lib_cpp dataStructure_lib utils_lib ${dependency_libs}) + +# Set C++ standard for C++ library +set_target_properties(cache_lib_cpp PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO +) -target_compile_options(cachelib PRIVATE -fPIC) -target_link_options(cachelib PRIVATE -Wl,--export-dynamic) diff --git a/libCacheSim/cache/admission/CMakeLists.txt b/libCacheSim/cache/admission/CMakeLists.txt deleted file mode 100644 index 960e680d..00000000 --- a/libCacheSim/cache/admission/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_library(admissionC prob.c size.c bloomfilter.c sizeProbabilistic.c) -add_library(admissionCpp adaptsize/adaptsize.cpp adaptsize/adaptsize_interface.cpp) -add_library(admission INTERFACE) -target_link_libraries(admission INTERFACE admissionC admissionCpp) \ No newline at end of file diff --git a/libCacheSim/cache/admission/adaptsize/adaptsize.h b/libCacheSim/cache/admission/adaptsize/adaptsize.h index 53c1afef..b9db2cb2 100644 --- a/libCacheSim/cache/admission/adaptsize/adaptsize.h +++ b/libCacheSim/cache/admission/adaptsize/adaptsize.h @@ -7,7 +7,7 @@ #include #include -#include "../../include/libCacheSim/request.h" +#include "libCacheSim/request.h" class Adaptsize { public: diff --git a/libCacheSim/cache/admission/adaptsize/adaptsize_interface.cpp b/libCacheSim/cache/admission/adaptsize/adaptsize_interface.cpp index 88769278..da9a4ea3 100644 --- a/libCacheSim/cache/admission/adaptsize/adaptsize_interface.cpp +++ b/libCacheSim/cache/admission/adaptsize/adaptsize_interface.cpp @@ -3,8 +3,8 @@ #include #include -#include "../../include/libCacheSim/admissionAlgo.h" -#include "adaptsize/adaptsize.h" +#include "adaptsize.h" +#include "libCacheSim/admissionAlgo.h" #ifdef __cplusplus extern "C" { diff --git a/libCacheSim/cache/eviction/CMakeLists.txt b/libCacheSim/cache/eviction/CMakeLists.txt deleted file mode 100644 index a221972d..00000000 --- a/libCacheSim/cache/eviction/CMakeLists.txt +++ /dev/null @@ -1,138 +0,0 @@ -set(sourceC - MRU.c - Random.c - RandomTwo.c - LFU.c - LFUDA.c - ARC.c - FIFO.c - LRU.c - Clock.c - ClockPro.c - SLRU.c - SLRUv0.c - CR_LFU.c - SR_LRU.c - Belady.c - BeladySize.c - Hyperbolic.c - LeCaR.c - Cacheus.c - TwoQ.c - ARCv0.c - LRUv0.c - LeCaRv0.c - LIRS.c - WTinyLFU.c - Size.c - LRUProb.c - QDLP.c - nop.c - - FIFO_Merge.c - FIFO_Reinsertion.c - - S3FIFO.c - S3FIFOv0.c - S3FIFOd.c - - other/flashProb.c - other/S3LRU.c - - Sieve.c - - CAR.c - - RandomLRU.c -) - -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/priv") - set(sourceC ${sourceC} - - priv/QDLPv0.c - - - priv/S3FIFOdv2.c - - priv/myMQv1.c - - priv/MClock.c - ) -endif() - -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/fifo") - set(sourceC ${sourceC} - fifo/LP_SFIFO.c - fifo/LP_TwoQ.c - fifo/LP_ARC.c - fifo/SFIFO.c - fifo/SFIFOv0.c - ) -endif() - -if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/belady") - set(sourceC ${sourceC} - belady/Sieve_Belady.c - belady/LRU_Belady.c - belady/FIFO_Belady.c - ) -endif() - -set(sourceCPP - cpp/LFU.cpp - cpp/GDSF.cpp - LHD/lhd.cpp - LHD/LHD_Interface.cpp - ) - -if (ENABLE_LRB) - set (sourceCPP ${sourceCPP} - LRB/lrb.cpp - LRB/LRB_Interface.cpp - ) -endif() - -if (ENABLE_3L_CACHE) - set (sourceCPP ${sourceCPP} - 3LCache/ThreeLCache_Interface.cpp - 3LCache/ThreeLCache.cpp - ) -endif() - -if (ENABLE_GLCACHE) - set(sourceC - ${sourceC} - GLCache/bucket.c - GLCache/dataPrep.c - GLCache/eviction.c - GLCache/GLCache.c - GLCache/inference.c - GLCache/init.c - GLCache/segment.c - GLCache/segSel.c - GLCache/train.c - ) -endif() - - -add_library (evictionC ${sourceC}) -target_link_libraries(evictionC cachelib dataStructure utils) -add_library (evictionCPP ${sourceCPP}) -target_link_libraries(evictionCPP cachelib utils) -set_target_properties(evictionCPP - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - ) - -# set (evictionLib -# evictionC -# evictionCPP -# ) - - - -# add_library(eviction INTERFACE) -# target_link_libraries(eviction INTERFACE ${evictionLib}) - diff --git a/libCacheSim/cache/eviction/RandomLRU.c b/libCacheSim/cache/eviction/RandomLRU.c index 370ce302..002a352e 100644 --- a/libCacheSim/cache/eviction/RandomLRU.c +++ b/libCacheSim/cache/eviction/RandomLRU.c @@ -21,6 +21,7 @@ extern "C" { typedef struct RandomLRU_params { int32_t n_samples; + cache_obj_t **eviction_candidates; } RandomLRU_params_t; static const char *DEFAULT_CACHE_PARAMS = "n-samples=16"; @@ -33,12 +34,14 @@ static const char *DEFAULT_CACHE_PARAMS = "n-samples=16"; static void RandomLRU_free(cache_t *cache); static bool RandomLRU_get(cache_t *cache, const request_t *req); -static cache_obj_t *RandomLRU_find(cache_t *cache, const request_t *req, const bool update_cache); +static cache_obj_t *RandomLRU_find(cache_t *cache, const request_t *req, + const bool update_cache); static cache_obj_t *RandomLRU_insert(cache_t *cache, const request_t *req); static cache_obj_t *RandomLRU_to_evict(cache_t *cache, const request_t *req); static void RandomLRU_evict(cache_t *cache, const request_t *req); static bool RandomLRU_remove(cache_t *cache, const obj_id_t obj_id); -static void RandomLRU_parse_params(cache_t *cache, const char *cache_specific_params); +static void RandomLRU_parse_params(cache_t *cache, + const char *cache_specific_params); // *********************************************************************** // **** **** @@ -52,11 +55,13 @@ static void RandomLRU_parse_params(cache_t *cache, const char *cache_specific_pa * @param ccache_params some common cache parameters * @param cache_specific_params RandomLRU specific parameters, should be NULL */ -cache_t *RandomLRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params) { +cache_t *RandomLRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params) { common_cache_params_t ccache_params_copy = ccache_params; ccache_params_copy.hashpower = MAX(12, ccache_params_copy.hashpower - 8); - cache_t *cache = cache_struct_init("RandomLRU", ccache_params_copy, cache_specific_params); + cache_t *cache = + cache_struct_init("RandomLRU", ccache_params_copy, cache_specific_params); cache->cache_init = RandomLRU_init; cache->cache_free = RandomLRU_free; cache->get = RandomLRU_get; @@ -66,7 +71,8 @@ cache_t *RandomLRU_init(const common_cache_params_t ccache_params, const char *c cache->evict = RandomLRU_evict; cache->remove = RandomLRU_remove; - cache->eviction_params = (RandomLRU_params_t *)malloc(sizeof(RandomLRU_params_t)); + cache->eviction_params = + (RandomLRU_params_t *)malloc(sizeof(RandomLRU_params_t)); RandomLRU_params_t *params = (RandomLRU_params_t *)(cache->eviction_params); memset(params, 0, sizeof(RandomLRU_params_t)); @@ -75,7 +81,11 @@ cache_t *RandomLRU_init(const common_cache_params_t ccache_params, const char *c RandomLRU_parse_params(cache, cache_specific_params); } - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "RandomLRU-%d", params->n_samples); + params->eviction_candidates = + (cache_obj_t **)malloc(sizeof(cache_obj_t *) * params->n_samples); + + snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "RandomLRU-%d", + params->n_samples); return cache; } @@ -85,7 +95,11 @@ cache_t *RandomLRU_init(const common_cache_params_t ccache_params, const char *c * * @param cache */ -static void RandomLRU_free(cache_t *cache) { cache_struct_free(cache); } +static void RandomLRU_free(cache_t *cache) { + RandomLRU_params_t *params = (RandomLRU_params_t *)(cache->eviction_params); + free(params->eviction_candidates); + cache_struct_free(cache); +} /** * @brief this function is the user facing API @@ -106,7 +120,9 @@ static void RandomLRU_free(cache_t *cache) { cache_struct_free(cache); } * @param req * @return true if cache hit, false if cache miss */ -static bool RandomLRU_get(cache_t *cache, const request_t *req) { return cache_get_base(cache, req); } +static bool RandomLRU_get(cache_t *cache, const request_t *req) { + return cache_get_base(cache, req); +} // *********************************************************************** // **** **** @@ -124,7 +140,8 @@ static bool RandomLRU_get(cache_t *cache, const request_t *req) { return cache_g * and if the object is expired, it is removed from the cache * @return true on hit, false on miss */ -static cache_obj_t *RandomLRU_find(cache_t *cache, const request_t *req, const bool update_cache) { +static cache_obj_t *RandomLRU_find(cache_t *cache, const request_t *req, + const bool update_cache) { cache_obj_t *obj = cache_find_base(cache, req, update_cache); if (obj != NULL && update_cache) { obj->Random.last_access_vtime = cache->n_req; @@ -170,7 +187,8 @@ static cache_obj_t *RandomLRU_to_evict(cache_t *cache, const request_t *req) { // } // qsort(obj_to_evict, N, sizeof(cache_obj_t *), compare); - // if (obj_to_evict1->Random.last_access_vtime < obj_to_evict2->Random.last_access_vtime) + // if (obj_to_evict1->Random.last_access_vtime < + // obj_to_evict2->Random.last_access_vtime) // return obj_to_evict1; // else // return obj_to_evict2; @@ -200,13 +218,13 @@ static int compare_access_time(const void *p1, const void *p2) { * @param req not used */ static void RandomLRU_evict(cache_t *cache, const request_t *req) { - const int N = 64; - cache_obj_t *obj_to_evict[N]; - for (int i = 0; i < N; i++) { - obj_to_evict[i] = hashtable_rand_obj(cache->hashtable); + RandomLRU_params_t *params = (RandomLRU_params_t *)(cache->eviction_params); + for (int i = 0; i < params->n_samples; i++) { + params->eviction_candidates[i] = hashtable_rand_obj(cache->hashtable); } - qsort(obj_to_evict, N, sizeof(cache_obj_t *), compare_access_time); - cache_evict_base(cache, obj_to_evict[0], true); + qsort(params->eviction_candidates, params->n_samples, sizeof(cache_obj_t *), + compare_access_time); + cache_evict_base(cache, params->eviction_candidates[0], true); } /** @@ -232,7 +250,8 @@ static bool RandomLRU_remove(cache_t *cache, const obj_id_t obj_id) { return true; } -static void RandomLRU_parse_params(cache_t *cache, const char *cache_specific_params) { +static void RandomLRU_parse_params(cache_t *cache, + const char *cache_specific_params) { RandomLRU_params_t *params = (RandomLRU_params_t *)(cache->eviction_params); char *params_str = strdup(cache_specific_params); diff --git a/libCacheSim/cache/eviction/other/S3LRU.c b/libCacheSim/cache/eviction/other/S3LRU.c index e82f4d11..f015e845 100644 --- a/libCacheSim/cache/eviction/other/S3LRU.c +++ b/libCacheSim/cache/eviction/other/S3LRU.c @@ -20,8 +20,8 @@ // Copyright © 2018 Juncheng. All rights reserved. // -#include "../../dataStructure/hashtable/hashtable.h" -#include "../../include/libCacheSim/evictionAlgo.h" +#include "../../../dataStructure/hashtable/hashtable.h" +#include "../../../include/libCacheSim/evictionAlgo.h" #ifdef __cplusplus extern "C" { diff --git a/libCacheSim/cache/eviction/priv/MClock.c b/libCacheSim/cache/eviction/priv/MClock.c deleted file mode 100644 index 76b1e9e0..00000000 --- a/libCacheSim/cache/eviction/priv/MClock.c +++ /dev/null @@ -1,479 +0,0 @@ -// -// multi handed clock -// there are m hands, each points to i/m position of the clock -// there is no action on hit, -// on miss, one of the hands is selected to evict based on next access distance -// (Belady) then the hands are reset to corresponding positions -// -// -// mClock.c -// libCacheSim -// -// Created by Juncheng on 12/4/18. -// Copyright © 2018 Juncheng. All rights reserved. -// - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/evictionAlgo.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - cache_obj_t *q_head; - cache_obj_t *q_tail; - cache_obj_t **hands; - int n_hands; - double *hand_pos; - int n_consider_obj; - - int last_eviction_hand; - - request_t *req_local; - -} MClock_params_t; - -static const char *DEFAULT_PARAMS = - "hand-pos=0:0.25:0.50:0.75:1.0,n-consider-obj=1"; -// static const char *DEFAULT_PARAMS = -// "hand-pos=0:0.10:0.20:0.30:0.40:0.50:0.60:0.70:0.80:0.90:1.0"; - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** - -static void MClock_parse_params(cache_t *cache, - const char *cache_specific_params); -static void MClock_free(cache_t *cache); -static bool MClock_get(cache_t *cache, const request_t *req); -static cache_obj_t *MClock_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *MClock_insert(cache_t *cache, const request_t *req); -static cache_obj_t *MClock_to_evict(cache_t *cache, const request_t *req); -static void MClock_evict(cache_t *cache, const request_t *req); -static bool MClock_remove(cache_t *cache, const obj_id_t obj_id); -static int64_t MClock_get_occupied_byte(const cache_t *cache); -static int64_t MClock_get_n_obj(const cache_t *cache); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// *********************************************************************** - -/** - * @brief initialize the cache - * - * @param ccache_params some common cache parameters - * @param cache_specific_params cache specific parameters, see parse_params - * function or use -e "print" with the cachesim binary - */ -cache_t *MClock_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = cache_struct_init("MClock", ccache_params, cache_specific_params); - cache->cache_init = MClock_init; - cache->cache_free = MClock_free; - cache->get = MClock_get; - cache->find = MClock_find; - cache->insert = MClock_insert; - cache->evict = MClock_evict; - cache->remove = MClock_remove; - cache->to_evict = MClock_to_evict; - cache->get_occupied_byte = MClock_get_occupied_byte; - cache->get_n_obj = MClock_get_n_obj; - cache->can_insert = cache_can_insert_default; - cache->obj_md_size = 0; - - cache->eviction_params = malloc(sizeof(MClock_params_t)); - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - params->q_head = NULL; - params->q_tail = NULL; - - params->req_local = new_request(); - - // we must parse the default params because the user specified params - // may not have hand-pos and the hand_pos array will not be initialized - MClock_parse_params(cache, DEFAULT_PARAMS); - if (cache_specific_params != NULL) { - MClock_parse_params(cache, cache_specific_params); - } - - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "MClock-%d-%d", - params->n_hands, params->n_consider_obj); - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void MClock_free(cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - free(params->hands); - free(params->hand_pos); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ -bool MClock_get(cache_t *cache, const request_t *req) { - return cache_get_base(cache, req); -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** -/** - * @brief find an object in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return the object or NULL if not found - */ -static cache_obj_t *MClock_find(cache_t *cache, const request_t *req, - const bool update_cache) { - cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - - return cache_obj; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * eviction should be - * performed before calling this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *MClock_insert(cache_t *cache, const request_t *req) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - cache_obj_t *obj = cache_insert_base(cache, req); - prepend_obj_to_head(¶ms->q_head, ¶ms->q_tail, obj); - - if (params->hands[0] != NULL) { - for (int i = 0; i < params->last_eviction_hand; i++) { - params->hands[i] = params->hands[i]->queue.prev; - } - if (params->last_eviction_hand == params->n_hands - 1) { - params->hands[params->last_eviction_hand] = params->q_tail; - } else { - params->hands[params->last_eviction_hand] = - params->hands[params->last_eviction_hand]->queue.prev; - } - assert(params->hands[0] != NULL); - assert(params->hands[0] == params->q_head); - } - - return obj; -} - -static void reset_hands(cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - cache_obj_t *obj = params->q_head; - int hand_idx = 0; - int64_t obj_idx = 0; - while (obj != NULL) { - if (obj_idx >= params->hand_pos[hand_idx] * cache->n_obj) { - params->hands[hand_idx++] = obj; - } - obj = obj->queue.next; - obj_idx++; - } - if (params->hand_pos[params->n_hands - 1] == 1) { - params->hands[hand_idx++] = params->q_tail; - } - assert(hand_idx == params->n_hands); -} - -static void print_curr_hand_pos(cache_t *cache) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - cache_obj_t *obj = params->q_head; - int hand_idx = 0; - int64_t obj_idx = 0; - while (obj != NULL) { - if (obj == params->hands[hand_idx]) { - hand_idx++; - } - obj = obj->queue.next; - obj_idx++; - } -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *MClock_to_evict(cache_t *cache, const request_t *req) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - int best_obj_pos = 0; - cache_obj_t *best_obj = params->hands[0]; - double best_obj_benefit = best_obj->misc.next_access_vtime; - for (int i = 0; i < params->n_hands; i++) { - if (params->hands[i]->misc.next_access_vtime > best_obj_benefit) { - best_obj_pos = i; - best_obj = params->hands[i]; - best_obj_benefit = best_obj->misc.next_access_vtime; - } - } - - cache->to_evict_candidate_gen_vtime = cache->n_req; - cache->to_evict_candidate = best_obj; - assert(false); - return best_obj; -} - -static double calc_candidate_score1(cache_t *cache, cache_obj_t *obj) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - double score = 0; - int n_score = 0; - cache_obj_t *curr_obj = obj; - - for (int i = 0; i < params->n_consider_obj; i++) { - if (curr_obj == NULL) { - break; - } - score += 1.0 / (curr_obj->misc.next_access_vtime - cache->n_req); - n_score++; - curr_obj = curr_obj->queue.prev; - } - - if (n_score > 0) score = n_score / score; - return score; -} - -static double calc_candidate_score(cache_t *cache, cache_obj_t *obj) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - double score = 0; - int n_score = 0; - cache_obj_t *curr_obj = obj; - - for (int i = 0; i < params->n_consider_obj; i++) { - if (curr_obj == NULL) { - break; - } - score += 1.0 / (curr_obj->misc.next_access_vtime - cache->n_req); - n_score++; - curr_obj = curr_obj->queue.prev; - } - - if (n_score > 0) score = n_score / score; - return score; -} -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void MClock_evict(cache_t *cache, const request_t *req) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - static __thread int n_reset_hand = 0; - if (params->hands[0] == NULL) { - n_reset_hand++; - assert(n_reset_hand < 2); - reset_hands(cache); - } - - int best_obj_pos = params->n_hands - 1; - cache_obj_t *best_obj = params->hands[params->n_hands - 1]; - double score = calc_candidate_score(cache, best_obj); - double best_score = score; - - /* 0.0 is the head (most recently inserted), - * 1.0 is the tail (least recently inserted) */ - for (int i = params->n_hands - 1; i >= 0; i--) { - score = calc_candidate_score(cache, params->hands[i]); - if (score > best_score) { - best_obj_pos = i; - best_obj = params->hands[i]; - best_score = score; - } - } - - - // static __thread int *n_choice_per_hand; - // static __thread int n_evictions = 0; - // if (n_choice_per_hand == NULL) { - // n_choice_per_hand = (int *)malloc(sizeof(int) * params->n_hands); - // memset(n_choice_per_hand, 0, sizeof(int) * params->n_hands); - // } - // n_choice_per_hand[best_obj_pos]++; - // if (++n_evictions % 1000 == 0) { - // for (int i = 0; i < params->n_hands; i++) { - // printf("%d ", n_choice_per_hand[i]); - // n_choice_per_hand[i] = 0; - // } - // printf("\n"); - // } - - params->last_eviction_hand = best_obj_pos; - params->hands[best_obj_pos] = best_obj->queue.next; - DEBUG_ASSERT(best_obj_pos == params->n_hands - 1 || - best_obj->queue.next != NULL); - - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, best_obj); - cache_evict_base(cache, best_obj, true); -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool MClock_remove(cache_t *cache, const obj_id_t obj_id) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; - } - - // need to update hands - assert(false); - remove_obj_from_list(¶ms->q_head, ¶ms->q_tail, obj); - cache_remove_obj_base(cache, obj, true); - - return true; -} - -static int64_t MClock_get_occupied_byte(const cache_t *cache) { - // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - int64_t occupied_byte = cache->occupied_byte; - return occupied_byte; -} - -static int64_t MClock_get_n_obj(const cache_t *cache) { - // MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - int64_t n_obj = cache->n_obj; - return n_obj; -} - -// *********************************************************************** -// **** **** -// **** parameter set up functions **** -// **** **** -// *********************************************************************** -static const char *MClock_current_params(cache_t *cache, - MClock_params_t *params) { - static __thread char params_str[128]; - int n = snprintf(params_str, 128, "hand-pos=%.2lf", (params->hand_pos[0])); - - for (int i = 1; i < params->n_hands; i++) { - n += snprintf(params_str + n, 128 - n, ":%.2lf", (params->hand_pos[i])); - } - - snprintf(cache->cache_name + n, 128 - n, "\n"); - - return params_str; -} - -static void MClock_parse_params(cache_t *cache, - const char *cache_specific_params) { - MClock_params_t *params = (MClock_params_t *)cache->eviction_params; - char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - char *end; - - while (params_str != NULL && params_str[0] != '\0') { - /* different parameters are separated by comma, - * key and value are separated by = */ - char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); - - // skip the white space - while (params_str != NULL && *params_str == ' ') { - params_str++; - } - - if (strcasecmp(key, "hand-pos") == 0) { - int n_hands = 0; -#define MClock_MAX_N_HAND 16 - double hand_pos[MClock_MAX_N_HAND]; - char *v = strsep((char **)&value, ":"); - while (v != NULL) { - hand_pos[n_hands++] = strtod(v, &end); - v = strsep((char **)&value, ":"); - } - params->n_hands = n_hands; - if (params->hands != NULL) { - free(params->hands); - } - if (params->hand_pos != NULL) { - free(params->hand_pos); - } - params->hands = calloc(params->n_hands, sizeof(cache_obj_t *)); - params->hand_pos = calloc(params->n_hands, sizeof(double)); - for (int i = 0; i < n_hands; i++) { - params->hand_pos[i] = hand_pos[i]; - } - assert(hand_pos[0] == 0); - assert(hand_pos[n_hands - 1] == 1); - } else if (strcasecmp(key, "n-consider-obj") == 0) { - params->n_consider_obj = (int)strtol(value, &end, 10); - } else if (strcasecmp(key, "print") == 0) { - printf("current parameters: %s\n", MClock_current_params(cache, params)); - exit(0); - } else { - ERROR("%s does not have parameter %s\n", cache->cache_name, key); - exit(1); - } - } - free(old_params_str); -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/eviction/priv/QDLPv0.c b/libCacheSim/cache/eviction/priv/QDLPv0.c deleted file mode 100644 index a6230d34..00000000 --- a/libCacheSim/cache/eviction/priv/QDLPv0.c +++ /dev/null @@ -1,512 +0,0 @@ -/** - * @file QDLP.c - * @brief Implementation of QDLPv0 eviction algorithm - * QDLPv0: lazy promotion and quick demotion - * it uses a probationary FIFO queue with a sieve cache - * objects are first inserted into the FIFO queue, and moved to the clock cache - * when evicting from the FIFO queue if it has been accessed - * if the cache is full, evict from the FIFO cache - * if the clock cache is full, then promoting to clock will trigger an eviction - * from the clock cache (not insert into the FIFO queue) - */ - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/cache.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// #define DEBUG_MODE - -// *********************************************************************** // -// **** debug macro **** // -#ifdef DEBUG_MODE -#define PRINT_CACHE_STATE(cache, params) \ - printf("cache size %ld + %ld + %ld, %ld %ld \t", params->n_fifo_byte, \ - params->n_fifo_ghost_byte, params->n_clock_byte, \ - cache->occupied_size, cache->cache_size); \ - // j -#define DEBUG_PRINT(FMT, ...) \ - do { \ - PRINT_CACHE_STATE(cache, params); \ - printf(FMT, ##__VA_ARGS__); \ - printf("%s", NORMAL); \ - fflush(stdout); \ - } while (0) - -#else -#define DEBUG_PRINT(FMT, ...) -#endif -// *********************************************************************** // - -typedef struct { - cache_obj_t *fifo_head; - cache_obj_t *fifo_tail; - - cache_obj_t *fifo_ghost_head; - cache_obj_t *fifo_ghost_tail; - - cache_obj_t *clock_head; - cache_obj_t *clock_tail; - int64_t n_fifo_obj; - int64_t n_fifo_byte; - int64_t n_fifo_ghost_obj; - int64_t n_fifo_ghost_byte; - int64_t n_clock_obj; - int64_t n_clock_byte; - // the user specified size - double fifo_ratio; - int64_t fifo_size; - int64_t clock_size; - int64_t fifo_ghost_size; - cache_obj_t *clock_pointer; - int64_t vtime_last_check_is_ghost; - - bool lazy_promotion; -} QDLPv0_params_t; - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** - -static void QDLPv0_parse_params(cache_t *cache, - const char *cache_specific_params); -static void QDLPv0_free(cache_t *cache); -static bool QDLPv0_get(cache_t *cache, const request_t *req); -static cache_obj_t *QDLPv0_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *QDLPv0_insert(cache_t *cache, const request_t *req); -static cache_obj_t *QDLPv0_to_evict(cache_t *cache, const request_t *req); -static void QDLPv0_evict(cache_t *cache, const request_t *req); -static bool QDLPv0_remove(cache_t *cache, const obj_id_t obj_id); - -/* internal functions */ -static void QDLPv0_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove); - -static void QDLPv0_clock_evict(cache_t *cache, const request_t *req); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// *********************************************************************** -/** - * @brief initialize a QDLPv0 cache - * - * @param ccache_params some common cache parameters - * @param cache_specific_params QDLPv0 specific parameters, should be NULL - */ -cache_t *QDLPv0_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = cache_struct_init("QDLPv0", ccache_params, cache_specific_params); - cache->cache_init = QDLPv0_init; - cache->cache_free = QDLPv0_free; - cache->get = QDLPv0_get; - cache->find = QDLPv0_find; - cache->insert = QDLPv0_insert; - cache->evict = QDLPv0_evict; - cache->remove = QDLPv0_remove; - cache->to_evict = QDLPv0_to_evict; - - if (ccache_params.consider_obj_metadata) { - cache->obj_md_size = 1; - } else { - cache->obj_md_size = 0; - } - - cache->eviction_params = my_malloc(QDLPv0_params_t); - QDLPv0_params_t *params = (QDLPv0_params_t *)cache->eviction_params; - memset(cache->eviction_params, 0, sizeof(QDLPv0_params_t)); - params->clock_pointer = NULL; - params->fifo_ratio = 0.2; - params->lazy_promotion = true; - - params->n_fifo_obj = 0; - params->n_fifo_byte = 0; - params->n_fifo_ghost_obj = 0; - params->n_fifo_ghost_byte = 0; - params->n_clock_obj = 0; - params->n_clock_byte = 0; - params->fifo_head = NULL; - params->fifo_tail = NULL; - params->clock_head = NULL; - params->clock_tail = NULL; - params->fifo_ghost_head = NULL; - params->fifo_ghost_tail = NULL; - params->vtime_last_check_is_ghost = -1; - - if (cache_specific_params != NULL) { - QDLPv0_parse_params(cache, cache_specific_params); - } - - params->fifo_size = cache->cache_size * params->fifo_ratio; - params->clock_size = cache->cache_size - params->fifo_size; - params->fifo_ghost_size = params->clock_size; - if (params->lazy_promotion) { - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "QDLPv0-%.4lf-lazy", - params->fifo_ratio); - } else { - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "QDLPv0-%.4lf", - params->fifo_ratio); - } - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void QDLPv0_free(cache_t *cache) { - free(cache->eviction_params); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ -static bool QDLPv0_get(cache_t *cache, const request_t *req) { - QDLPv0_params_t *params = cache->eviction_params; - bool cache_hit = cache_get_base(cache, req); - DEBUG_PRINT("%ld QDLPv0_get2\n", cache->n_req); - DEBUG_ASSERT(params->n_fifo_obj + params->n_clock_obj == cache->n_obj); - DEBUG_ASSERT(params->n_fifo_byte + params->n_clock_byte == - cache->occupied_byte); - - return cache_hit; -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** -/** - * @brief check whether an object is in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return true on hit, false on miss - */ -static cache_obj_t *QDLPv0_find(cache_t *cache, const request_t *req, - const bool update_cache) { - QDLPv0_params_t *params = cache->eviction_params; - cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - cache_obj_t *ret = cache_obj; - bool cache_hit = (cache_obj != NULL && cache_obj->QDLP.cache_id != 3); - DEBUG_PRINT("%ld QDLPv0_find %s\n", cache->n_req, cache_hit ? "hit" : "miss"); - - if (cache_obj != NULL && update_cache) { - if (cache_obj->QDLP.cache_id == 1) { - cache_obj->QDLP.freq += 1; - if (!params->lazy_promotion) { - /* FIFO cache, promote to clock cache */ - remove_obj_from_list(¶ms->fifo_head, ¶ms->fifo_tail, cache_obj); - params->n_fifo_obj--; - params->n_fifo_byte -= cache_obj->obj_size + cache->obj_md_size; - prepend_obj_to_head(¶ms->clock_head, ¶ms->clock_tail, - cache_obj); - params->n_clock_obj++; - params->n_clock_byte += cache_obj->obj_size + cache->obj_md_size; - cache_obj->QDLP.cache_id = 2; - cache_obj->QDLP.freq = 0; - while (params->n_clock_byte > params->clock_size) { - // clock cache is full, evict from clock cache - QDLPv0_clock_evict(cache, req); - } - } - } else if (cache_obj->QDLP.cache_id == 2) { - // clock cache - DEBUG_PRINT("%ld QDLPv0_find hit on clock\n", cache->n_req); - if (cache_obj->QDLP.freq < 1) { - // using one-bit, using multi-bit reduce miss ratio most of the time - cache_obj->QDLP.freq += 1; - } - } else if (cache_obj->QDLP.cache_id == 3) { - // FIFO ghost - DEBUG_PRINT("%ld QDLPv0_check ghost\n", cache->n_req); - DEBUG_ASSERT(cache_hit == false); - params->vtime_last_check_is_ghost = cache->n_req; - QDLPv0_remove_obj(cache, cache_obj); - } else { - ERROR("cache_obj->QDLP.cache_id = %d", cache_obj->QDLP.cache_id); - } - } - - if (cache_hit) - return ret; - else - return NULL; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * and eviction is not part of this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *QDLPv0_insert(cache_t *cache, const request_t *req) { - QDLPv0_params_t *params = cache->eviction_params; - - cache_obj_t *obj = cache_insert_base(cache, req); - obj->QDLP.visited = false; - - if (cache->n_req == params->vtime_last_check_is_ghost) { - // insert to Clock - DEBUG_PRINT("%ld QDLPv0_insert to clock\n", cache->n_req); - prepend_obj_to_head(¶ms->clock_head, ¶ms->clock_tail, obj); - params->n_clock_obj++; - params->n_clock_byte += obj->obj_size + cache->obj_md_size; - obj->QDLP.cache_id = 2; - obj->QDLP.freq = 0; - - } else { - // insert to FIFO - DEBUG_PRINT("%ld QDLPv0_insert to fifo\n", cache->n_req); - prepend_obj_to_head(¶ms->fifo_head, ¶ms->fifo_tail, obj); - params->n_fifo_obj++; - params->n_fifo_byte += obj->obj_size + cache->obj_md_size; - obj->QDLP.cache_id = 1; - obj->QDLP.freq = 0; - } - - return obj; -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *QDLPv0_to_evict(cache_t *cache, const request_t *req) { - QDLPv0_params_t *params = cache->eviction_params; - if (params->fifo_tail != NULL) { - return params->fifo_tail; - } else { - // not implemented, we need to evict from the clock cache - assert(0); - } - return NULL; -} - -/** - * @brief evict an object from the clock - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void QDLPv0_clock_evict(cache_t *cache, const request_t *reqj) { - QDLPv0_params_t *params = cache->eviction_params; - cache_obj_t *pointer = params->clock_pointer; - - /* if we have run one full around or first eviction */ - if (pointer == NULL) { - pointer = params->clock_tail; - } - - /* find the first untouched */ - int n_loop = 0; - while (pointer->QDLP.freq > 0) { - pointer->QDLP.freq -= 1; - pointer = pointer->queue.prev; - if (pointer == NULL) { - n_loop += 1; - pointer = params->clock_tail; - assert(n_loop < 1000); - } - } - - params->clock_pointer = pointer->queue.prev; - remove_obj_from_list(¶ms->clock_head, ¶ms->clock_tail, pointer); - params->n_clock_obj--; - params->n_clock_byte -= pointer->obj_size + cache->obj_md_size; - cache_evict_base(cache, pointer, true); -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void QDLPv0_evict(cache_t *cache, const request_t *req) { - QDLPv0_params_t *params = cache->eviction_params; - - if (params->n_fifo_byte > params->fifo_size || params->n_clock_obj == 0) { - DEBUG_PRINT("%ld QDLPv0_evict_fifo\n", cache->n_req); - cache_obj_t *obj_to_evict = params->fifo_tail; - DEBUG_ASSERT(obj_to_evict != NULL); - - // remove from FIFO - remove_obj_from_list(¶ms->fifo_head, ¶ms->fifo_tail, obj_to_evict); - params->n_fifo_obj--; - params->n_fifo_byte -= obj_to_evict->obj_size + cache->obj_md_size; - - if (params->lazy_promotion && obj_to_evict->QDLP.freq > 0) { - prepend_obj_to_head(¶ms->clock_head, ¶ms->clock_tail, - obj_to_evict); - params->n_clock_obj++; - params->n_clock_byte += obj_to_evict->obj_size + cache->obj_md_size; - obj_to_evict->QDLP.cache_id = 2; - obj_to_evict->QDLP.freq = 0; - while (params->n_clock_byte > params->clock_size) { - // clock cache is full, evict from clock cache - QDLPv0_clock_evict(cache, req); - } - } else { - cache_evict_base(cache, obj_to_evict, false); - prepend_obj_to_head(¶ms->fifo_ghost_head, ¶ms->fifo_ghost_tail, - obj_to_evict); - params->n_fifo_ghost_obj++; - params->n_fifo_ghost_byte += obj_to_evict->obj_size + cache->obj_md_size; - obj_to_evict->QDLP.cache_id = 3; - while (params->n_fifo_ghost_byte > params->fifo_ghost_size) { - // clock cache is full, evict from clock cache - QDLPv0_remove_obj(cache, params->fifo_ghost_tail); - } - } - } else { - DEBUG_PRINT("%ld QDLPv0_evict_clock\n", cache->n_req); - // evict from clock cache, this can happen because we insert to the clock - // when the object hit on the ghost - QDLPv0_clock_evict(cache, req); - } -} - -static void QDLPv0_remove_obj(cache_t *cache, cache_obj_t *obj_to_remove) { - DEBUG_ASSERT(obj_to_remove != NULL); - QDLPv0_params_t *params = cache->eviction_params; - - if (obj_to_remove->QDLP.cache_id == 1) { - // fifo cache - remove_obj_from_list(¶ms->fifo_head, ¶ms->fifo_tail, obj_to_remove); - params->n_fifo_obj--; - params->n_fifo_byte -= obj_to_remove->obj_size + cache->obj_md_size; - cache_remove_obj_base(cache, obj_to_remove, true); - } else if (obj_to_remove->QDLP.cache_id == 2) { - // clock cache - remove_obj_from_list(¶ms->clock_head, ¶ms->clock_tail, - obj_to_remove); - params->n_clock_obj--; - params->n_clock_byte -= obj_to_remove->obj_size + cache->obj_md_size; - cache_remove_obj_base(cache, obj_to_remove, true); - } else if (obj_to_remove->QDLP.cache_id == 3) { - // fifo ghost - remove_obj_from_list(¶ms->fifo_ghost_head, ¶ms->fifo_ghost_tail, - obj_to_remove); - params->n_fifo_ghost_obj--; - params->n_fifo_ghost_byte -= obj_to_remove->obj_size + cache->obj_md_size; - hashtable_delete(cache->hashtable, obj_to_remove); - } -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool QDLPv0_remove(cache_t *cache, const obj_id_t obj_id) { - cache_obj_t *obj = hashtable_find_obj_id(cache->hashtable, obj_id); - if (obj == NULL) { - return false; - } - QDLPv0_remove_obj(cache, obj); - - return true; -} - -// *********************************************************************** -// **** **** -// **** parameter set up functions **** -// **** **** -// *********************************************************************** -static const char *QDLPv0_current_params(QDLPv0_params_t *params) { - static __thread char params_str[128]; - snprintf(params_str, 128, "fifo-ratio=%.4lf\n", params->fifo_ratio); - return params_str; -} - -static void QDLPv0_parse_params(cache_t *cache, - const char *cache_specific_params) { - QDLPv0_params_t *params = (QDLPv0_params_t *)cache->eviction_params; - char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - char *end; - - while (params_str != NULL && params_str[0] != '\0') { - /* different parameters are separated by comma, - * key and value are separated by = */ - char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); - - // skip the white space - while (params_str != NULL && *params_str == ' ') { - params_str++; - } - - if (strcasecmp(key, "fifo-ratio") == 0) { - params->fifo_ratio = (double)strtof(value, &end); - if (strlen(end) > 2) { - ERROR("param parsing error, find string \"%s\" after number\n", end); - } - - } else if (strcasecmp(key, "print") == 0) { - printf("current parameters: %s\n", QDLPv0_current_params(params)); - exit(0); - } else { - ERROR("%s does not have parameter %s\n", cache->cache_name, key); - exit(1); - } - } - free(old_params_str); -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c b/libCacheSim/cache/eviction/priv/S3FIFOdv2.c deleted file mode 100644 index 4e51f113..00000000 --- a/libCacheSim/cache/eviction/priv/S3FIFOdv2.c +++ /dev/null @@ -1,880 +0,0 @@ -// -// 10% small FIFO + 90% main FIFO (2-bit Clock) + ghost -// insert to small FIFO if not in the ghost, else insert to the main FIFO -// evict from small FIFO: -// if object in the small is accessed, -// reinsert to main FIFO, -// else -// evict and insert to the ghost -// evict from main FIFO: -// if object in the main is accessed, -// reinsert to main FIFO, -// else -// evict -// -// -// S3FIFOdv2.c -// libCacheSim -// -// Created by Juncheng on 12/4/18. -// Copyright © 2018 Juncheng. All rights reserved. -// - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/evictionAlgo.h" -#include "../../../utils/include/mymath.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define PROB_ARRAY_SIZE 10000000ULL -#define MAX_FREQ_THRESHOLD 8 - -#define MAX_FREQ_CNT 32 - -typedef struct { - cache_t *fifo; - cache_t *fifo_ghost; - cache_t *main_cache; - bool hit_on_ghost; - - int64_t n_obj_admit_to_fifo; - int64_t n_obj_admit_to_main; - int64_t n_obj_move_to_main; - int64_t n_byte_admit_to_fifo; - int64_t n_byte_admit_to_main; - int64_t n_byte_move_to_main; - - int64_t n_obj_admit_to_fifo_last; - int64_t n_obj_admit_to_main_last; - int64_t n_obj_move_to_main_last; - - int32_t *first_access_cnt_over_age_fifo; - int32_t *first_access_cnt_over_age_main; - int64_t n_insertion; - int32_t age_granularity; - - int move_to_main_threshold; - int64_t eviction_freq_cnt[MAX_FREQ_THRESHOLD]; - int threshold_cnt[MAX_FREQ_THRESHOLD]; - - int64_t main_freq_cnt[MAX_FREQ_CNT]; - int main_reinsert_threshold; - - int64_t small_freq_cnt[MAX_FREQ_CNT]; - int small_reinsert_threshold; - - double fifo_size_ratio; - double ghost_size_ratio; - char main_cache_type[32]; - - request_t *req_local; -} S3FIFOdv2_params_t; - -static const char *DEFAULT_CACHE_PARAMS = - "fifo-size-ratio=0.10,ghost-size-ratio=0.90,move-to-main-threshold=1"; - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** -static void S3FIFOdv2_free(cache_t *cache); -static bool S3FIFOdv2_get(cache_t *cache, const request_t *req); - -static bool S3FIFOdv2_remove(cache_t *cache, const obj_id_t obj_id); -static inline int64_t S3FIFOdv2_get_occupied_byte(const cache_t *cache); -static inline int64_t S3FIFOdv2_get_n_obj(const cache_t *cache); -static inline bool S3FIFOdv2_can_insert(cache_t *cache, const request_t *req); -static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req); -static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req); -static void S3FIFOdv2_rebalance(cache_t *cache); -static void S3FIFOdv2_choose_threshold(cache_t *cache); -static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache); -static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache); -static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache); - -static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req); -static cache_obj_t *S3FIFOdv2_to_evict(cache_t *cache, const request_t *req); -static void S3FIFOdv2_evict(cache_t *cache, const request_t *req); -static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// *********************************************************************** - -cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = - cache_struct_init("S3FIFOdv2", ccache_params, cache_specific_params); - cache->cache_init = S3FIFOdv2_init; - cache->cache_free = S3FIFOdv2_free; - cache->get = S3FIFOdv2_get; - cache->find = S3FIFOdv2_find; - cache->insert = S3FIFOdv2_insert; - cache->evict = S3FIFOdv2_evict; - cache->remove = S3FIFOdv2_remove; - cache->to_evict = S3FIFOdv2_to_evict; - cache->get_n_obj = S3FIFOdv2_get_n_obj; - cache->get_occupied_byte = S3FIFOdv2_get_occupied_byte; - cache->can_insert = S3FIFOdv2_can_insert; - - cache->obj_md_size = 0; - - cache->eviction_params = malloc(sizeof(S3FIFOdv2_params_t)); - memset(cache->eviction_params, 0, sizeof(S3FIFOdv2_params_t)); - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - params->first_access_cnt_over_age_fifo = - malloc(sizeof(int32_t) * PROB_ARRAY_SIZE); - params->first_access_cnt_over_age_main = - malloc(sizeof(int32_t) * PROB_ARRAY_SIZE); - memset(params->first_access_cnt_over_age_fifo, 0, - sizeof(int32_t) * PROB_ARRAY_SIZE); - memset(params->first_access_cnt_over_age_main, 0, - sizeof(int32_t) * PROB_ARRAY_SIZE); - - params->age_granularity = - (int32_t)ceil((double)ccache_params.cache_size * 2 / PROB_ARRAY_SIZE); - - params->req_local = new_request(); - params->hit_on_ghost = false; - memset(params->main_freq_cnt, 0, sizeof(int64_t) * MAX_FREQ_CNT); - params->main_reinsert_threshold = 1; - memset(params->small_freq_cnt, 0, sizeof(int64_t) * MAX_FREQ_CNT); - params->small_reinsert_threshold = 1; - - S3FIFOdv2_parse_params(cache, DEFAULT_CACHE_PARAMS); - if (cache_specific_params != NULL) { - S3FIFOdv2_parse_params(cache, cache_specific_params); - } - - int64_t fifo_cache_size = - (int64_t)ccache_params.cache_size * params->fifo_size_ratio; - int64_t main_cache_size = ccache_params.cache_size - fifo_cache_size; - int64_t fifo_ghost_cache_size = - (int64_t)(ccache_params.cache_size * params->ghost_size_ratio); - - common_cache_params_t ccache_params_local = ccache_params; - ccache_params_local.cache_size = fifo_cache_size; - params->fifo = FIFO_init(ccache_params_local, NULL); - - if (fifo_ghost_cache_size > 0) { - ccache_params_local.cache_size = fifo_ghost_cache_size; - params->fifo_ghost = FIFO_init(ccache_params_local, NULL); - snprintf(params->fifo_ghost->cache_name, CACHE_NAME_ARRAY_LEN, - "FIFO-ghost"); - } else { - params->fifo_ghost = NULL; - } - - ccache_params_local.cache_size = main_cache_size; - params->main_cache = FIFO_init(ccache_params_local, NULL); - -#if defined(TRACK_EVICTION_V_AGE) - if (params->fifo_ghost != NULL) { - params->fifo_ghost->track_eviction_age = false; - } - params->fifo->track_eviction_age = false; - params->main_cache->track_eviction_age = false; -#endif - - snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "S3FIFOdv2-%.4lf-%d", - params->fifo_size_ratio, params->move_to_main_threshold); - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void S3FIFOdv2_free(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - free_request(params->req_local); - params->fifo->cache_free(params->fifo); - if (params->fifo_ghost != NULL) { - params->fifo_ghost->cache_free(params->fifo_ghost); - } - params->main_cache->cache_free(params->main_cache); - free(cache->eviction_params); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ -static bool S3FIFOdv2_get(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - DEBUG_ASSERT(params->fifo->get_occupied_byte(params->fifo) + - params->main_cache->get_occupied_byte(params->main_cache) <= - cache->cache_size); - - bool cache_hit = cache_get_base(cache, req); - - if (params->main_cache->get_occupied_byte(params->main_cache) > - 0.9 * params->main_cache->cache_size && - cache->n_req % 1000 == 0) { - // S3FIFOdv2_rebalance(cache); - // S3FIFOdv2_choose_threshold(cache); - S3FIFOdv2_update_main_reinsert_threshold(cache); - S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache); - // S3FIFOdv2_update_small_reinsert_threshold(cache); - } - - // if (cache->n_req % 1000000 == 0) { - // double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + - // params->n_obj_move_to_main; - // printf("\n#################### %ld %ld + %ld = %ld %d\n", cache->n_req, - // params->fifo->cache_size, params->main_cache->cache_size, - // cache->cache_size, params->move_to_main_threshold); - // printf("%ld %ld: %.4lf/%.4lf/%.4lf\n", cache->n_req, cache->cache_size, - // (double)params->n_obj_admit_to_fifo / s, - // (double)params->n_obj_admit_to_main / s, - // (double)params->n_obj_move_to_main / s); - // } - - return cache_hit; -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** -static void S3FIFOdv2_update_main_reinsert_threshold_incorrect(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - int n_main_obj = params->main_cache->get_n_obj(params->main_cache); - double inserttion_frac = 0.1; - int64_t n_drop_obj = n_main_obj * (1 - inserttion_frac); - int64_t n_obj = 0; - - const int64_t POWER2_MAPPING[] = { - 1, 2, 4, 8, 16, 32, 64, - 128, 256, 512, 1024, 2048, 4096, 8192, - 16384, 32768, 65536, 131072, 262144, 524288, 1048576, - 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, - 268435456, 536870912, 1073741824}; - - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->main_freq_cnt[i]; - if (n_obj > n_drop_obj) { - params->move_to_main_threshold = POWER2_MAPPING[i]; - break; - } - } - - n_obj = 0; - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->main_freq_cnt[i]; - } - if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); - - // if (params->move_to_main_threshold > 1) { - // printf("set threshold to %d, ", params->move_to_main_threshold); - // for (int i = 0; i < 8; i++) { - // printf("%ld ", params->main_freq_cnt[i]); - // } - // printf("\n"); - // } -} - -static void S3FIFOdv2_update_main_reinsert_threshold(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - int n_main_obj = params->main_cache->get_n_obj(params->main_cache); - double inserttion_frac = 0.1; - int64_t n_drop_obj = n_main_obj * (1 - inserttion_frac); - int64_t n_obj = 0; - - const int64_t POWER2_MAPPING[] = { - 1, 2, 4, 8, 16, 32, 64, - 128, 256, 512, 1024, 2048, 4096, 8192, - 16384, 32768, 65536, 131072, 262144, 524288, 1048576, - 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, - 268435456, 536870912, 1073741824}; - - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->main_freq_cnt[i]; - if (n_obj > n_drop_obj) { - params->main_reinsert_threshold = POWER2_MAPPING[i]; - break; - } - } - // printf("%ld %ld %ld %ld - %ld/%d\n", params->main_freq_cnt[0], - // params->main_freq_cnt[1], params->main_freq_cnt[2], - // params->main_freq_cnt[3], n_drop_obj, n_main_obj); - - n_obj = 0; - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->main_freq_cnt[i]; - } - if (n_obj != n_main_obj) - ERROR("n_obj %ld != n_main_obj %d\n", (long)n_obj, n_main_obj); - - // if (params->main_reinsert_threshold > 1) { - // printf("set threshold to %d, ", params->main_reinsert_threshold); - // for (int i = 0; i < 8; i++) { - // printf("%ld ", params->main_freq_cnt[i]); - // } - // printf("\n"); - // } -} - -static void S3FIFOdv2_update_small_reinsert_threshold(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - int n_small_obj = params->fifo->get_n_obj(params->fifo); - double inserttion_frac = 0.2; - int64_t n_drop_obj = n_small_obj * (1 - inserttion_frac); - int64_t n_obj = 0; - - const int64_t POWER2_MAPPING[] = { - 1, 2, 4, 8, 16, 32, 64, - 128, 256, 512, 1024, 2048, 4096, 8192, - 16384, 32768, 65536, 131072, 262144, 524288, 1048576, - 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, - 268435456, 536870912, 1073741824}; - - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->small_freq_cnt[i]; - if (n_obj > n_drop_obj) { - params->move_to_main_threshold = POWER2_MAPPING[i]; - break; - } - } - // printf("%ld %ld %ld %ld - %ld/%ld\n", params->small_freq_cnt[0], - // params->small_freq_cnt[1], params->small_freq_cnt[2], - // params->small_freq_cnt[3], n_drop_obj, n_small_obj); - - n_obj = 0; - for (int i = 0; i < MAX_FREQ_CNT; i++) { - n_obj += params->small_freq_cnt[i]; - } - if (n_obj != n_small_obj) - ERROR("n_obj %ld != n_small_obj %d\n", (long)n_obj, n_small_obj); - - // if (params->move_to_main_threshold > 1) { - // printf("set threshold to %d, ", params->move_to_main_threshold); - // for (int i = 0; i < 8; i++) { - // printf("%ld ", params->small_freq_cnt[i]); - // } - // printf("\n"); - // } -} - -static void S3FIFOdv2_choose_threshold(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - int64_t sum_cnt = 0; - for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - sum_cnt += params->eviction_freq_cnt[i]; - } - - if (sum_cnt < 10000) { - return; - } - - int64_t sum_cnt2 = sum_cnt; - for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - sum_cnt2 -= params->eviction_freq_cnt[i]; - if (sum_cnt2 < sum_cnt / 5) { - params->move_to_main_threshold = i + 1; - break; - } - } - - params->threshold_cnt[params->move_to_main_threshold - 1]++; - for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - params->eviction_freq_cnt[i] /= 2; - } - - if (params->move_to_main_threshold != 1) { - for (int i = 0; i < MAX_FREQ_THRESHOLD; i++) { - printf("%8ld/%-4d ", (long)params->eviction_freq_cnt[i], - (int)params->threshold_cnt[i]); - } - - printf("move_to_main_threshold: %d\n", params->move_to_main_threshold); - } -} - -static void S3FIFOdv2_rebalance(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - - // double s = params->n_obj_admit_to_fifo + params->n_obj_admit_to_main + - // params->n_obj_move_to_main; - - size_t n_slots = - MIN(params->fifo->cache_size, params->main_cache->cache_size) / 100 + 1; - - if ((double)(params->n_obj_admit_to_main - params->n_obj_admit_to_main_last) / - (params->n_obj_admit_to_fifo - params->n_obj_admit_to_fifo_last) > - 0.1) { - if ((double)(params->fifo->cache_size) / (double)(cache->cache_size) > - 0.10 && - params->move_to_main_threshold > 1) { - params->move_to_main_threshold -= 1; - } - params->fifo->cache_size += n_slots; - params->main_cache->cache_size -= n_slots; - params->fifo_ghost->cache_size -= n_slots; - } - if ((double)(params->n_obj_move_to_main - params->n_obj_move_to_main_last) / - (params->n_obj_admit_to_fifo - params->n_obj_admit_to_fifo_last) > - 0.1) { - if ((double)(params->fifo->cache_size) / (double)(cache->cache_size) < - 0.001) { - params->move_to_main_threshold += 1; - } - params->fifo->cache_size -= n_slots; - params->main_cache->cache_size += n_slots; - params->fifo_ghost->cache_size += n_slots; - } - - params->n_obj_admit_to_fifo_last = params->n_obj_admit_to_fifo; - params->n_obj_admit_to_main_last = params->n_obj_admit_to_main; - params->n_obj_move_to_main_last = params->n_obj_move_to_main; -} - -/** - * @brief find an object in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return the object or NULL if not found - */ -static cache_obj_t *S3FIFOdv2_find(cache_t *cache, const request_t *req, - const bool update_cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - - // if update cache is false, we only check the fifo and main caches - if (!update_cache) { - cache_obj_t *obj = params->fifo->find(params->fifo, req, false); - if (obj != NULL) { - return obj; - } - obj = params->main_cache->find(params->main_cache, req, false); - if (obj != NULL) { - return obj; - } - return NULL; - } - - /* update cache is true from now */ - params->hit_on_ghost = false; - cache_obj_t *obj = params->fifo->find(params->fifo, req, true); - if (obj != NULL) { - obj->S3FIFO.freq += 1; - size_t pos1 = log2_ull(obj->S3FIFO.freq - 1); - size_t pos2 = log2_ull(obj->S3FIFO.freq); - params->small_freq_cnt[pos1] -= 1; // remove from old freq - params->small_freq_cnt[pos2] += 1; // add to new freq - - if (obj->misc.freq == 1) { - size_t insertion_age = params->n_insertion - obj->S3FIFO.insertion_time; - if (insertion_age / params->age_granularity >= PROB_ARRAY_SIZE) { - insertion_age = PROB_ARRAY_SIZE - 1; - } - params->first_access_cnt_over_age_fifo[insertion_age / - params->age_granularity] += 1; - } - return obj; - } - - if (params->fifo_ghost != NULL && - params->fifo_ghost->remove(params->fifo_ghost, req->obj_id)) { - // if object in fifo_ghost, remove will return true - params->hit_on_ghost = true; - } - - obj = params->main_cache->find(params->main_cache, req, true); - if (obj != NULL) { - obj->S3FIFO.freq += 1; - size_t pos1 = log2_ull(obj->S3FIFO.freq - 1); - size_t pos2 = log2_ull(obj->S3FIFO.freq); - params->main_freq_cnt[pos1] -= 1; // remove from old freq - params->main_freq_cnt[pos2] += 1; // add to new freq - - // this wont happen - if (obj->misc.freq - obj->S3FIFO.main_insert_freq == 1) { - size_t insertion_age = params->n_insertion - obj->S3FIFO.insertion_time; - if (insertion_age / params->age_granularity >= PROB_ARRAY_SIZE) { - insertion_age = PROB_ARRAY_SIZE - 1; - } - params->first_access_cnt_over_age_main[insertion_age / - params->age_granularity] += 1; - } - } - - return obj; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * eviction should be - * performed before calling this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *S3FIFOdv2_insert(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - cache_obj_t *obj = NULL; - - if (params->hit_on_ghost) { - /* insert into the ARC */ - params->hit_on_ghost = false; - params->n_obj_admit_to_main += 1; - params->n_byte_admit_to_main += req->obj_size; - obj = params->main_cache->insert(params->main_cache, req); - obj->S3FIFO.main_insert_freq = 0; - params->main_freq_cnt[0] += 1; - } else { - /* insert into the fifo */ - if (req->obj_size >= params->fifo->cache_size) { - return NULL; - } - params->n_obj_admit_to_fifo += 1; - params->n_byte_admit_to_fifo += req->obj_size; - obj = params->fifo->insert(params->fifo, req); - params->small_freq_cnt[0] += 1; - } - - obj->S3FIFO.insertion_time = params->n_insertion++; - -#if defined(TRACK_EVICTION_V_AGE) - obj->create_time = CURR_TIME(cache, req); -#endif - -#if defined(TRACK_DEMOTION) - obj->create_time = cache->n_req; -#endif - - obj->S3FIFO.freq = 0; - - return obj; -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *S3FIFOdv2_to_evict(cache_t *cache, const request_t *req) { - assert(false); - return NULL; -} - -static void S3FIFOdv2_evict_fifo(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - cache_t *fifo = params->fifo; - cache_t *ghost = params->fifo_ghost; - cache_t *main = params->main_cache; - - bool has_evicted = false; - while (!has_evicted && fifo->get_occupied_byte(fifo) > 0) { - // evict from FIFO - cache_obj_t *obj_to_evict = fifo->to_evict(fifo, req); - DEBUG_ASSERT(obj_to_evict != NULL); - // need to copy the object before it is evicted - copy_cache_obj_to_request(params->req_local, obj_to_evict); - size_t pos = MIN(obj_to_evict->S3FIFO.freq, MAX_FREQ_THRESHOLD - 1); - params->eviction_freq_cnt[pos] += 1; - - params->small_freq_cnt[log2_ull(obj_to_evict->S3FIFO.freq)] -= 1; - - if (obj_to_evict->S3FIFO.freq >= params->move_to_main_threshold) { -#if defined(TRACK_DEMOTION) - printf("%ld keep %ld %ld\n", cache->n_req, obj_to_evict->create_time, - obj_to_evict->misc.next_access_vtime); -#endif - // freq is updated in cache_find_base - params->n_obj_move_to_main += 1; - params->n_byte_move_to_main += obj_to_evict->obj_size; - - cache_obj_t *new_obj = main->insert(main, params->req_local); - new_obj->misc.freq = obj_to_evict->misc.freq; - new_obj->S3FIFO.freq = 0; - new_obj->S3FIFO.main_insert_freq = obj_to_evict->misc.freq; - params->main_freq_cnt[0] += 1; -#if defined(TRACK_EVICTION_V_AGE) - new_obj->create_time = obj_to_evict->create_time; - } else { - record_eviction_age(cache, obj_to_evict, - CURR_TIME(cache, req) - obj_to_evict->create_time); -#else - } else { -#endif - -#if defined(TRACK_DEMOTION) - printf("%ld demote %ld %ld\n", cache->n_req, obj_to_evict->create_time, - obj_to_evict->misc.next_access_vtime); -#endif - - // insert to ghost - if (ghost != NULL) { - ghost->get(ghost, params->req_local); - } - has_evicted = true; - } - - // remove from fifo, but do not update stat - bool removed = fifo->remove(fifo, params->req_local->obj_id); - assert(removed); - } -} - -static void S3FIFOdv2_evict_main(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - // cache_t *fifo = params->fifo; - // cache_t *ghost = params->fifo_ghost; - cache_t *main = params->main_cache; - - // evict from main cache - bool has_evicted = false; - while (!has_evicted && main->get_occupied_byte(main) > 0) { - cache_obj_t *obj_to_evict = main->to_evict(main, req); - DEBUG_ASSERT(obj_to_evict != NULL); - int freq = obj_to_evict->S3FIFO.freq; - int total_freq = obj_to_evict->misc.freq; -#if defined(TRACK_EVICTION_V_AGE) - int64_t create_time = obj_to_evict->create_time; -#endif - copy_cache_obj_to_request(params->req_local, obj_to_evict); - params->main_freq_cnt[log2_ull(freq)] -= 1; - if (freq >= params->main_reinsert_threshold) { - // we need to evict first because the object to insert has the same obj_id - main->remove(main, obj_to_evict->obj_id); - obj_to_evict = NULL; - - cache_obj_t *new_obj = main->insert(main, params->req_local); - // clock with 2-bit counter - new_obj->S3FIFO.freq = MIN(freq, 3) - 1; - new_obj->misc.freq = total_freq; - params->main_freq_cnt[log2_ull(new_obj->S3FIFO.freq)] += 1; - -#if defined(TRACK_EVICTION_V_AGE) - new_obj->create_time = create_time; -#endif - } else { -#if defined(TRACK_EVICTION_V_AGE) - record_eviction_age(cache, obj_to_evict, - CURR_TIME(cache, req) - obj_to_evict->create_time); -#endif - - // main->evict(main, req); - bool removed = main->remove(main, obj_to_evict->obj_id); - if (!removed) { - ERROR("cannot remove obj %ld\n", (long)obj_to_evict->obj_id); - } - - has_evicted = true; - } - } -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void S3FIFOdv2_evict(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - - cache_t *fifo = params->fifo; - // cache_t *ghost = params->fifo_ghost; - cache_t *main = params->main_cache; - - if (main->get_occupied_byte(main) > main->cache_size || - fifo->get_occupied_byte(fifo) == 0) { - S3FIFOdv2_evict_main(cache, req); - } else { - S3FIFOdv2_evict_fifo(cache, req); - } -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool S3FIFOdv2_remove(cache_t *cache, const obj_id_t obj_id) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - bool removed = false; - removed = removed || params->fifo->remove(params->fifo, obj_id); - removed = removed || (params->fifo_ghost && - params->fifo_ghost->remove(params->fifo_ghost, obj_id)); - removed = removed || params->main_cache->remove(params->main_cache, obj_id); - - return removed; -} - -static inline int64_t S3FIFOdv2_get_occupied_byte(const cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - return params->fifo->get_occupied_byte(params->fifo) + - params->main_cache->get_occupied_byte(params->main_cache); -} - -static inline int64_t S3FIFOdv2_get_n_obj(const cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - return params->fifo->get_n_obj(params->fifo) + - params->main_cache->get_n_obj(params->main_cache); -} - -static inline bool S3FIFOdv2_can_insert(cache_t *cache, const request_t *req) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - - return req->obj_size <= params->fifo->cache_size && cache_can_insert_default(cache, req); -} - -// *********************************************************************** -// **** **** -// **** parameter set up functions **** -// **** **** -// *********************************************************************** -static const char *S3FIFOdv2_current_params(S3FIFOdv2_params_t *params) { - static __thread char params_str[128]; - snprintf(params_str, 128, "fifo-size-ratio=%.4lf,main-cache=%s\n", - params->fifo_size_ratio, params->main_cache->cache_name); - return params_str; -} - -static void S3FIFOdv2_parse_params(cache_t *cache, - const char *cache_specific_params) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)(cache->eviction_params); - - char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - // char *end; - - while (params_str != NULL && params_str[0] != '\0') { - /* different parameters are separated by comma, - * key and value are separated by = */ - char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); - - // skip the white space - while (params_str != NULL && *params_str == ' ') { - params_str++; - } - - if (strcasecmp(key, "fifo-size-ratio") == 0) { - params->fifo_size_ratio = strtod(value, NULL); - } else if (strcasecmp(key, "ghost-size-ratio") == 0) { - params->ghost_size_ratio = strtod(value, NULL); - } else if (strcasecmp(key, "move-to-main-threshold") == 0) { - params->move_to_main_threshold = atoi(value); - } else if (strcasecmp(key, "print") == 0) { - printf("parameters: %s\n", S3FIFOdv2_current_params(params)); - exit(0); - } else { - ERROR("%s does not have parameter %s\n", cache->cache_name, key); - exit(1); - } - } - - free(old_params_str); -} - -static bool S3FIFOdv2_rebalance0(cache_t *cache) { - S3FIFOdv2_params_t *params = (S3FIFOdv2_params_t *)cache->eviction_params; - - int64_t sum_fifo = 0; - int64_t sum_main = 0; - for (size_t i = 0; i < PROB_ARRAY_SIZE; i++) { - sum_fifo += params->first_access_cnt_over_age_fifo[i]; - sum_main += params->first_access_cnt_over_age_main[i]; - } - - int64_t sum_one_perc_fifo = 0, sum_one_perc_main = 0; - size_t pos_fifo = params->fifo->cache_size / params->age_granularity; - size_t pos_main = params->main_cache->cache_size / params->age_granularity; - size_t n_slots = - MIN(params->fifo->cache_size, params->main_cache->cache_size) / 20 + 1; - for (size_t i = 0; i < n_slots; i++) { - sum_one_perc_fifo += params->first_access_cnt_over_age_fifo[pos_fifo - i]; - sum_one_perc_main += params->first_access_cnt_over_age_main[pos_main - i]; - } - - if (sum_one_perc_fifo + sum_one_perc_main < 100) { - return false; - } - - // for (int i = 0; i < 10; i++) { - // printf("%4d ", params->first_access_cnt_over_age_fifo[i]); - // } - // printf("%ld - %ld\n", sum_fifo, sum_one_perc_fifo); - - // for (int i = 0; i < 10; i++) { - // printf("%4d ", params->first_access_cnt_over_age_main[i]); - // } - // printf("%ld - %ld\n", sum_main, sum_one_perc_main); - - n_slots = n_slots / 5; - if (sum_one_perc_fifo > sum_one_perc_main * 2) { - params->fifo->cache_size += n_slots; - params->main_cache->cache_size -= n_slots; - params->fifo_ghost->cache_size -= n_slots; - // printf("move to fifo %ld\n", n_slots); - } else if (sum_one_perc_main > sum_one_perc_fifo * 2) { - params->fifo->cache_size -= n_slots; - params->main_cache->cache_size += n_slots; - params->fifo_ghost->cache_size += n_slots; - // printf("move to main %ld\n", n_slots); - } - - memset(params->first_access_cnt_over_age_fifo, 0, - sizeof(*params->first_access_cnt_over_age_fifo)); - memset(params->first_access_cnt_over_age_main, 0, - sizeof(*params->first_access_cnt_over_age_main)); - - return true; -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/eviction/priv/myMQv1.c b/libCacheSim/cache/eviction/priv/myMQv1.c deleted file mode 100644 index 0dde2cc7..00000000 --- a/libCacheSim/cache/eviction/priv/myMQv1.c +++ /dev/null @@ -1,369 +0,0 @@ -// -// multi queue -// each queue has a specified size and a specified ghost size -// promotion upon hit -// -// in progress, this is not finished -// -// -// myMQv1.c -// libCacheSim -// -// Created by Juncheng on 12/4/18. -// Copyright © 2018 Juncheng. All rights reserved. -// - -#include "../../../dataStructure/hashtable/hashtable.h" -#include "../../../include/libCacheSim/evictionAlgo.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - cache_t **caches; - cache_t **ghost_caches; - int64_t n_caches; - int64_t *cache_sizes; - int64_t *ghost_sizes; - - request_t *req_local; - -} myMQv1_params_t; - -static const char *DEFAULT_PARAMS = - "cache_sizes=0.25,0.25,0.25,0.25;ghost_sizes=0.75,0.75,0.75,0.75"; -#define myMQv1_MAX_N_cache 16 - -// *********************************************************************** -// **** **** -// **** function declarations **** -// **** **** -// *********************************************************************** - -static void myMQv1_parse_params(cache_t *cache, - const char *cache_specific_params); -static void myMQv1_free(cache_t *cache); -static bool myMQv1_get(cache_t *cache, const request_t *req); -static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, - const bool update_cache); -static cache_obj_t *myMQv1_insert(cache_t *cache, const request_t *req); -static cache_obj_t *myMQv1_to_evict(cache_t *cache, const request_t *req); -static void myMQv1_evict(cache_t *cache, const request_t *req); -static bool myMQv1_remove(cache_t *cache, const obj_id_t obj_id); -static int64_t myMQv1_get_occupied_byte(const cache_t *cache); -static int64_t myMQv1_get_n_obj(const cache_t *cache); - -// *********************************************************************** -// **** **** -// **** end user facing functions **** -// **** **** -// *********************************************************************** - -/** - * @brief initialize the cache - * - * @param ccache_params some common cache parameters - * @param cache_specific_params cache specific parameters, see parse_params - * function or use -e "print" with the cachesim binary - */ -cache_t *myMQv1_init(const common_cache_params_t ccache_params, - const char *cache_specific_params) { - cache_t *cache = - cache_struct_init("myMQv1", ccache_params, cache_specific_params); - cache->cache_init = myMQv1_init; - cache->cache_free = myMQv1_free; - cache->get = myMQv1_get; - cache->find = myMQv1_find; - cache->insert = myMQv1_insert; - cache->evict = myMQv1_evict; - cache->remove = myMQv1_remove; - cache->to_evict = myMQv1_to_evict; - cache->get_occupied_byte = myMQv1_get_occupied_byte; - cache->get_n_obj = myMQv1_get_n_obj; - cache->can_insert = cache_can_insert_default; - cache->obj_md_size = 0; - - cache->eviction_params = malloc(sizeof(myMQv1_params_t)); - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - params->req_local = new_request(); - - if (cache_specific_params != NULL) { - myMQv1_parse_params(cache, cache_specific_params); - } else { - myMQv1_parse_params(cache, DEFAULT_PARAMS); - } - - params->caches = malloc(sizeof(cache_t *) * params->n_caches); - params->ghost_caches = malloc(sizeof(cache_t *) * params->n_caches); - common_cache_params_t ccache_params_local = ccache_params; - for (int i = 0; i < params->n_caches; i++) { - ccache_params_local.cache_size = params->cache_sizes[i]; - params->caches[i] = FIFO_init(ccache_params, NULL); - params->ghost_caches[i] = FIFO_init(ccache_params_local, NULL); - } - - // snprintf(cache->cache_name, CACHE_NAME_ARRAY_LEN, "myMQv1", 0.25); - - return cache; -} - -/** - * free resources used by this cache - * - * @param cache - */ -static void myMQv1_free(cache_t *cache) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - for (int i = 0; i < params->n_caches; i++) { - params->caches[i]->cache_free(params->caches[i]); - params->ghost_caches[i]->cache_free(params->ghost_caches[i]); - } - free(params->caches); - free(params->ghost_caches); - free(params->cache_sizes); - free(cache->eviction_params); - cache_struct_free(cache); -} - -/** - * @brief this function is the user facing API - * it performs the following logic - * - * ``` - * if obj in cache: - * update_metadata - * return true - * else: - * if cache does not have enough space: - * evict until it has space to insert - * insert the object - * return false - * ``` - * - * @param cache - * @param req - * @return true if cache hit, false if cache miss - */ -bool myMQv1_get(cache_t *cache, const request_t *req) { - return cache_get_base(cache, req); -} - -// *********************************************************************** -// **** **** -// **** developer facing APIs (used by cache developer) **** -// **** **** -// *********************************************************************** -/** - * @brief find an object in the cache - * - * @param cache - * @param req - * @param update_cache whether to update the cache, - * if true, the object is promoted - * and if the object is expired, it is removed from the cache - * @return the object or NULL if not found - */ -static cache_obj_t *myMQv1_find(cache_t *cache, const request_t *req, - const bool update_cache) { - cache_obj_t *cache_obj = cache_find_base(cache, req, update_cache); - ERROR("todo"); - - return cache_obj; -} - -/** - * @brief insert an object into the cache, - * update the hash table and cache metadata - * this function assumes the cache has enough space - * eviction should be - * performed before calling this function - * - * @param cache - * @param req - * @return the inserted object - */ -static cache_obj_t *myMQv1_insert(cache_t *cache, const request_t *req) { - // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - - cache_obj_t *obj = cache_insert_base(cache, req); - ERROR("todo"); - - return obj; -} - -/** - * @brief find the object to be evicted - * this function does not actually evict the object or update metadata - * not all eviction algorithms support this function - * because the eviction logic cannot be decoupled from finding eviction - * candidate, so use assert(false) if you cannot support this function - * - * @param cache the cache - * @return the object to be evicted - */ -static cache_obj_t *myMQv1_to_evict(cache_t *cache, const request_t *req) { - assert(false); - return NULL; -} - -/** - * @brief evict an object from the cache - * it needs to call cache_evict_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param req not used - * @param evicted_obj if not NULL, return the evicted object to caller - */ -static void myMQv1_evict(cache_t *cache, const request_t *req) { - // myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - - ERROR("todo"); -} - -/** - * @brief remove an object from the cache - * this is different from cache_evict because it is used to for user trigger - * remove, and eviction is used by the cache to make space for new objects - * - * it needs to call cache_remove_obj_base before returning - * which updates some metadata such as n_obj, occupied size, and hash table - * - * @param cache - * @param obj_id - * @return true if the object is removed, false if the object is not in the - * cache - */ -static bool myMQv1_remove(cache_t *cache, const obj_id_t obj_id) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - - for (int i = 0; i < params->n_caches; i++) { - if (params->caches[i]->remove(params->caches[i], obj_id)) { - return true; - } - } - - return true; -} - -static int64_t myMQv1_get_occupied_byte(const cache_t *cache) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - int64_t occupied_byte = 0; - for (int i = 0; i < params->n_caches; i++) { - occupied_byte += params->caches[i]->get_occupied_byte(params->caches[i]); - } - return occupied_byte; -} - -static int64_t myMQv1_get_n_obj(const cache_t *cache) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - int64_t n_obj = 0; - for (int i = 0; i < params->n_caches; i++) { - n_obj += params->caches[i]->get_n_obj(params->caches[i]); - } - return n_obj; -} - -// *********************************************************************** -// **** **** -// **** parameter set up functions **** -// **** **** -// *********************************************************************** -static const char *myMQv1_current_params(cache_t *cache, - myMQv1_params_t *params) { - static __thread char params_str[128]; - int n = snprintf(params_str, 128, "n-caches=%ld;cache-size-ratio=%d", - (long)params->n_caches, - (int)(params->cache_sizes[0] * 100 / cache->cache_size)); - - for (int i = 1; i < params->n_caches; i++) { - n += snprintf(params_str + n, 128 - n, ":%d", - (int)(params->cache_sizes[i] * 100 / cache->cache_size)); - } - n += snprintf(params_str + n, 128 - n, ";ghost-size-ratio=%d", - (int)(params->ghost_sizes[0] * 100 / cache->cache_size)); - for (int i = 1; i < params->n_caches; i++) { - n += snprintf(params_str + n, 128 - n, ":%d", - (int)(params->ghost_sizes[i] * 100 / cache->cache_size)); - } - - snprintf(cache->cache_name + n, 128 - n, "\n"); - - return params_str; -} - -static void myMQv1_parse_params(cache_t *cache, - const char *cache_specific_params) { - myMQv1_params_t *params = (myMQv1_params_t *)cache->eviction_params; - char *params_str = strdup(cache_specific_params); - char *old_params_str = params_str; - char *end; - - while (params_str != NULL && params_str[0] != '\0') { - /* different parameters are separated by comma, - * key and value are separated by = */ - char *key = strsep((char **)¶ms_str, "="); - char *value = strsep((char **)¶ms_str, ","); - - // skip the white space - while (params_str != NULL && *params_str == ' ') { - params_str++; - } - - if (strcasecmp(key, "n-cache") == 0) { - params->n_caches = (int)strtol(value, &end, 0); - if (strlen(end) > 2) { - ERROR("param parsing error, find string \"%s\" after number\n", end); - } - } else if (strcasecmp(key, "cache-size-ratio") == 0) { - int n_caches = 0; - int64_t cache_size_sum = 0; - int64_t cache_size_array[myMQv1_MAX_N_cache]; - char *v = strsep((char **)&value, ":"); - while (v != NULL) { - cache_size_array[n_caches++] = (int64_t)strtol(v, &end, 0); - cache_size_sum += cache_size_array[n_caches - 1]; - v = strsep((char **)&value, ":"); - } - if (params->n_caches != 0 && params->n_caches != n_caches) { - ERROR("n-cache and n-ghost-cache must be the same\n"); - exit(1); - } - params->n_caches = n_caches; - params->cache_sizes = calloc(params->n_caches, sizeof(int64_t)); - for (int i = 0; i < n_caches; i++) { - params->cache_sizes[i] = (int64_t)((double)cache_size_array[i] / - cache_size_sum * cache->cache_size); - } - } else if (strcasecmp(key, "ghost-size-ratio") == 0) { - int n_ghost_caches = 0; - int64_t ghost_size_array[myMQv1_MAX_N_cache]; - char *v = strsep((char **)&value, ":"); - while (v != NULL) { - ghost_size_array[n_ghost_caches++] = (int64_t)strtol(v, &end, 0); - v = strsep((char **)&value, ":"); - } - if (params->n_caches != 0 && params->n_caches != n_ghost_caches) { - ERROR("n-cache and n-ghost-cache must be the same\n"); - exit(1); - } - params->ghost_sizes = calloc(params->n_caches, sizeof(int64_t)); - for (int i = 0; i < n_ghost_caches; i++) { - params->ghost_sizes[i] = - (int64_t)((double)ghost_size_array[i] / cache->cache_size); - } - } else if (strcasecmp(key, "print") == 0) { - printf("current parameters: %s\n", myMQv1_current_params(cache, params)); - exit(0); - } else { - ERROR("%s does not have parameter %s\n", cache->cache_name, key); - exit(1); - } - } - free(old_params_str); -} - -#ifdef __cplusplus -} -#endif diff --git a/libCacheSim/cache/prefetch/CMakeLists.txt b/libCacheSim/cache/prefetch/CMakeLists.txt deleted file mode 100644 index e41aa166..00000000 --- a/libCacheSim/cache/prefetch/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ - -add_library(prefetchC Mithril.c OBL.c PG.c) - -add_library(prefetch INTERFACE) -target_link_libraries(prefetch INTERFACE prefetchC) - - diff --git a/libCacheSim/dataStructure/CMakeLists.txt b/libCacheSim/dataStructure/CMakeLists.txt index f1193e1b..2b002afd 100644 --- a/libCacheSim/dataStructure/CMakeLists.txt +++ b/libCacheSim/dataStructure/CMakeLists.txt @@ -1,12 +1,32 @@ -add_subdirectory(hash) -set(source - pqueue.c - splay.c - bloom.c - minimalIncrementCBF.c - hash/murmur3.c - hashtable/chainedHashtable.c - hashtable/chainedHashTableV2.c - ) -add_library (dataStructure ${source}) +# ============================================================================== +# Data Structure Library +# ============================================================================== +set(dataStructure_sources_c + pqueue.c + splay.c + bloom.c + minimalIncrementCBF.c + histogram.c + splay_tuple.c + hash/murmur3.c + hashtable/chainedHashtable.c + hashtable/chainedHashTableV2.c +) + +add_library(dataStructure_lib ${dataStructure_sources_c}) + +# Set target-specific compiler flags +target_compile_options(dataStructure_lib PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} +) + +target_include_directories(dataStructure_lib PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + + +set(dataStructure_sources ${dataStructure_sources_c} PARENT_SCOPE) \ No newline at end of file diff --git a/libCacheSim/include/libCacheSim/evictionAlgo.h b/libCacheSim/include/libCacheSim/evictionAlgo.h index 6eeef8b2..4ae92edf 100644 --- a/libCacheSim/include/libCacheSim/evictionAlgo.h +++ b/libCacheSim/include/libCacheSim/evictionAlgo.h @@ -38,125 +38,161 @@ typedef struct { int64_t n_byte_rewritten; } Clock_params_t; -cache_t *ARC_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *ARC_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *ARCv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *ARCv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Belady_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Belady_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *BeladySize_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *BeladySize_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Cacheus_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Cacheus_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Clock_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Clock_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *ClockPro_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *ClockPro_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *CR_LFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *CR_LFU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *FIFO_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *FIFO_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *GDSF_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *GDSF_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Hyperbolic_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Hyperbolic_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LeCaR_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LeCaR_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LeCaRv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LeCaRv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LFU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LFUCpp_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LFUCpp_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LFUDA_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LFUDA_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LHD_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LHD_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LRUv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LRUv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *MRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *MRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Random_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Random_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *RandomTwo_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *RandomTwo_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *SLRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *SLRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *SLRUv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *SLRUv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *SR_LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *SR_LRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *TwoQ_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *TwoQ_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LIRS_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LIRS_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Size_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Size_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *WTinyLFU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *FIFO_Merge_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *FIFO_Merge_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *FIFO_Reinsertion_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *FIFO_Reinsertion_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *flashProb_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *flashProb_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LRU_Prob_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LRU_Prob_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *SFIFOv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *SFIFO_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *SFIFO_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *nop_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *nop_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *QDLP_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *QDLP_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *S3LRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *S3LRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *S3FIFO_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *S3FIFO_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *S3FIFOv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *S3FIFOv0_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *S3FIFOd_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *S3FIFOd_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Sieve_Belady_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Sieve_Belady_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *LRU_Belady_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *LRU_Belady_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *FIFO_Belady_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *FIFO_Belady_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *Sieve_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *Sieve_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *RandomLRU_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *RandomLRU_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -cache_t *CAR_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *ThreeLCache_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *CAR_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); -#ifdef ENABLE_LRB -cache_t *LRB_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +#ifdef ENABLE_3L_CACHE +cache_t *ThreeLCache_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); #endif -#ifdef INCLUDE_PRIV -cache_t *LP_SFIFO_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *LP_ARC_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *LP_TwoQ_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *QDLPv0_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *S3FIFOdv2_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *myMQv1_init(const common_cache_params_t ccache_params, const char *cache_specific_params); - -cache_t *MClock_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +#ifdef ENABLE_LRB +cache_t *LRB_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); #endif -#if defined(ENABLE_GLCACHE) && ENABLE_GLCACHE == 1 +#if defined(ENABLE_GLCACHE) -cache_t *GLCache_init(const common_cache_params_t ccache_params, const char *cache_specific_params); +cache_t *GLCache_init(const common_cache_params_t ccache_params, + const char *cache_specific_params); #endif diff --git a/libCacheSim/mrcProfiler/CMakeLists.txt b/libCacheSim/mrcProfiler/CMakeLists.txt index 3245510a..96e3acee 100644 --- a/libCacheSim/mrcProfiler/CMakeLists.txt +++ b/libCacheSim/mrcProfiler/CMakeLists.txt @@ -1,11 +1,34 @@ +# ============================================================================== +# MRC Profiler Library +# ============================================================================== -file(GLOB source *.cpp) +# Collect all source files +set(mrcProfiler_sources_cpp + mrcProfiler.cpp +) + +add_library(mrcProfiler_lib ${mrcProfiler_sources_cpp}) -add_library (mrcProfilerLib ${source} ${source2}) +# Set C++ standard +set_target_properties(mrcProfiler_lib PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO +) -set_target_properties(mrcProfilerLib - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO +# Set target-specific compiler flags +target_compile_options(mrcProfiler_lib PRIVATE + ${LIBCACHESIM_CXX_FLAGS} + ${tcmalloc_flags} ) + +target_include_directories(mrcProfiler_lib PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +# Link dependencies +target_link_libraries(mrcProfiler_lib utils_lib traceReader_lib profiler_lib cache_lib_c cache_lib_cpp dataStructure_lib ${dependency_libs}) + +set(mrcProfiler_sources ${mrcProfiler_sources_cpp} PARENT_SCOPE) diff --git a/libCacheSim/profiler/CMakeLists.txt b/libCacheSim/profiler/CMakeLists.txt index c2b5ff40..293e8a87 100644 --- a/libCacheSim/profiler/CMakeLists.txt +++ b/libCacheSim/profiler/CMakeLists.txt @@ -1,7 +1,27 @@ -aux_source_directory(. DIR_LIB_SRCS) -add_library (profiler ${DIR_LIB_SRCS}) -target_link_libraries(profiler traceReader) +# ============================================================================== +# Profiler Library +# ============================================================================== -#file(GLOB src *.c) -#add_library (profiler ${src}) -#set(LIB_SOURCE ${LIB_SOURCE} ${src} PARENT_SCOPE) +set(profiler_sources_c + profilerLRU.c + simulator.c + dist.c +) + +add_library(profiler_lib ${profiler_sources_c}) + +# Set target-specific compiler flags +target_compile_options(profiler_lib PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} +) + +target_include_directories(profiler_lib PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +target_link_libraries(profiler_lib cache_lib_c cache_lib_cpp traceReader_lib utils_lib dataStructure_lib ${dependency_libs}) + +set(profiler_sources ${profiler_sources_c} PARENT_SCOPE) diff --git a/libCacheSim/traceAnalyzer/CMakeLists.txt b/libCacheSim/traceAnalyzer/CMakeLists.txt index 68bf8d05..4fc4e40c 100644 --- a/libCacheSim/traceAnalyzer/CMakeLists.txt +++ b/libCacheSim/traceAnalyzer/CMakeLists.txt @@ -1,13 +1,35 @@ +# ============================================================================== +# Trace Analyzer Library +# ============================================================================== -file(GLOB source *.cpp) -file(GLOB source2 experimental/*.cpp) -add_library (traceAnalyzerLib ${source} ${source2}) +# Collect all analyzer source files +set(traceAnalyzer_sources_cpp + utils/threadPool.cpp + utils/utils.cpp + utils/utilsSys.cpp -set_target_properties(traceAnalyzerLib - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO + accessPattern.cpp + analyzer.cpp + popularity.cpp + popularityDecay.cpp + reqRate.cpp + reuse.cpp + size.cpp + ttl.cpp + + experimental/createFutureReuseCCDF.cpp + experimental/lifetime.cpp + experimental/probAtAge.cpp + experimental/sizeChange.cpp +) + +add_library(traceAnalyzer_lib ${traceAnalyzer_sources_cpp}) +target_link_libraries(traceAnalyzer_lib ${dependency_libs}) +set_target_properties(traceAnalyzer_lib PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO ) +set(traceAnalyzer_sources ${traceAnalyzer_sources_cpp} PARENT_SCOPE) diff --git a/libCacheSim/traceAnalyzer/utils/CMakeLists.txt b/libCacheSim/traceAnalyzer/utils/CMakeLists.txt deleted file mode 100644 index 9c11a603..00000000 --- a/libCacheSim/traceAnalyzer/utils/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -#aux_source_directory(. DIR_LIB_SRCS) -#add_library (utils ${DIR_LIB_SRCS}) -#target_link_libraries(utils traceReader) - - -set (SOURCE utils.cpp utilsSys.cpp threadPool.cpp) -add_library (utils ${SOURCE}) -#target_link_libraries(utils traceReader) - - - diff --git a/libCacheSim/traceAnalyzer/utils/include/utilsStr.h b/libCacheSim/traceAnalyzer/utils/include/utilsStr.h index a604e2d1..043a792c 100644 --- a/libCacheSim/traceAnalyzer/utils/include/utilsStr.h +++ b/libCacheSim/traceAnalyzer/utils/include/utilsStr.h @@ -4,11 +4,12 @@ // Created by Juncheng Yang on 6/19/20. // -#include "const.h" -#include -#include -#include +#include +#include +#include +#include +#include "libCacheSim/const.h" namespace utilsStr { @@ -19,26 +20,25 @@ namespace utilsStr { * @param str a 8 byte char array */ static inline void convert_size_to_str(long long size, char *str) { - - if (size >= TiB) { - sprintf(str, "%.0lf TiB", (double) size / TiB); - } else if (size >= GiB) { - sprintf(str, "%.0lf GiB", (double) size / GiB); - } else if (size >= MiB) { - sprintf(str, "%.0lf MiB", (double) size / MiB); - } else if (size >= KiB) { - sprintf(str, "%.0lf KiB", (double) size / KiB); - } else { - sprintf(str, "%lld B", size); - } + if (size >= TiB) { + sprintf(str, "%.0lf TiB", (double)size / TiB); + } else if (size >= GiB) { + sprintf(str, "%.0lf GiB", (double)size / GiB); + } else if (size >= MiB) { + sprintf(str, "%.0lf MiB", (double)size / MiB); + } else if (size >= KiB) { + sprintf(str, "%.0lf KiB", (double)size / KiB); + } else { + sprintf(str, "%lld B", size); + } } static inline uint64_t str_to_u64(const char *start, size_t len) { - uint64_t n = 0; - while (len--) { - n = n * 10 + *start++ - '0'; - } - return n; + uint64_t n = 0; + while (len--) { + n = n * 10 + *start++ - '0'; + } + return n; } -}// namespace utilsStr \ No newline at end of file +} // namespace utilsStr \ No newline at end of file diff --git a/libCacheSim/traceAnalyzer/utils/utils.cpp b/libCacheSim/traceAnalyzer/utils/utils.cpp index 79b2daf5..33ba62b5 100644 --- a/libCacheSim/traceAnalyzer/utils/utils.cpp +++ b/libCacheSim/traceAnalyzer/utils/utils.cpp @@ -2,8 +2,8 @@ // Created by Juncheng Yang on 10/16/20. // -#include "utils.h" -#include "utilsMath.h" -#include "utilsPrint.h" -#include "utilsStr.h" -//#include "utilsSys.h" +#include "include/utils.h" + +#include "include/utilsMath.h" +#include "include/utilsPrint.h" +#include "include/utilsStr.h" diff --git a/libCacheSim/traceAnalyzer/utils/utilsSys.cpp b/libCacheSim/traceAnalyzer/utils/utilsSys.cpp index ce586802..ebe932ad 100644 --- a/libCacheSim/traceAnalyzer/utils/utilsSys.cpp +++ b/libCacheSim/traceAnalyzer/utils/utilsSys.cpp @@ -1,13 +1,14 @@ -#include "utilsSys.h" -#include "logging.h" -#include -#include -#include +#include "include/utilsSys.h" + #include #include +#include +#include #include +#include +#include "libCacheSim/logging.h" #ifdef __linux__ #include @@ -15,36 +16,37 @@ int utilsSys::set_thread_affinity(pthread_t tid) { #ifdef __linux__ - static int last_core_id = -1; - int num_cores = sysconf(_SC_NPROCESSORS_ONLN); + static int last_core_id = -1; + int num_cores = sysconf(_SC_NPROCESSORS_ONLN); - last_core_id++; - last_core_id %= num_cores; - DEBUG("assign thread affinity %d/%d\n", last_core_id, num_cores); + last_core_id++; + last_core_id %= num_cores; + DEBUG("assign thread affinity %d/%d\n", last_core_id, num_cores); - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(last_core_id, &cpuset); + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(last_core_id, &cpuset); - int rc = pthread_setaffinity_np(tid, sizeof(cpu_set_t), &cpuset); - if (rc != 0) { - WARN("Error calling pthread_setaffinity_np: %d\n", rc); - } + int rc = pthread_setaffinity_np(tid, sizeof(cpu_set_t), &cpuset); + if (rc != 0) { + WARN("Error calling pthread_setaffinity_np: %d\n", rc); + } #endif - return 0; + return 0; } int utilsSys::get_n_cores() { #ifdef __linux__ - INFO("This system has %d processors configured and " - "%d processors available.\n", - get_nprocs_conf(), get_nprocs()); + INFO( + "This system has %d processors configured and " + "%d processors available.\n", + get_nprocs_conf(), get_nprocs()); - return get_nprocs(); + return get_nprocs(); #else -// WARN("non linux system, use 4 threads as default\n"); - return static_cast(std::thread::hardware_concurrency()); + // WARN("non linux system, use 4 threads as default\n"); + return static_cast(std::thread::hardware_concurrency()); #endif } @@ -65,18 +67,17 @@ int utilsSys::get_n_available_cores() { } void utilsSys::print_cwd() { - char cwd[1024]; - if (getcwd(cwd, sizeof(cwd)) != nullptr) { - printf("Current working dir: %s\n", cwd); - } else { - perror("getcwd() error"); - } + char cwd[1024]; + if (getcwd(cwd, sizeof(cwd)) != nullptr) { + printf("Current working dir: %s\n", cwd); + } else { + perror("getcwd() error"); + } } - #ifdef HAS_GLIB void print_glib_ver(void) { - printf("glib version %d.%d.%d %d\n", glib_major_version, glib_minor_version, - glib_micro_version, glib_binary_age); + printf("glib version %d.%d.%d %d\n", glib_major_version, glib_minor_version, + glib_micro_version, glib_binary_age); } #endif diff --git a/libCacheSim/traceReader/CMakeLists.txt b/libCacheSim/traceReader/CMakeLists.txt index ec2fceeb..2e7dc10a 100644 --- a/libCacheSim/traceReader/CMakeLists.txt +++ b/libCacheSim/traceReader/CMakeLists.txt @@ -1,5 +1,8 @@ +# ============================================================================== +# Trace Reader Library +# ============================================================================== -set(source +set(traceReader_sources_c generalReader/binary.c generalReader/csv.c generalReader/txt.c @@ -11,9 +14,24 @@ set(source ) if (OPT_SUPPORT_ZSTD_TRACE) - set(source ${source} generalReader/zstdReader.c) + set(traceReader_sources_c ${traceReader_sources_c} generalReader/zstdReader.c) endif (OPT_SUPPORT_ZSTD_TRACE) -add_library(traceReader ${source}) +add_library(traceReader_lib ${traceReader_sources_c}) +# Set target-specific compiler flags +target_compile_options(traceReader_lib PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} +) + +target_include_directories(traceReader_lib PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +target_link_libraries(traceReader_lib utils_lib ${ZSTD_LIBRARIES}) + +set(traceReader_sources ${traceReader_sources_c} PARENT_SCOPE) diff --git a/libCacheSim/utils/CMakeLists.txt b/libCacheSim/utils/CMakeLists.txt index 595bdbf7..d0775ebd 100644 --- a/libCacheSim/utils/CMakeLists.txt +++ b/libCacheSim/utils/CMakeLists.txt @@ -1,7 +1,26 @@ -aux_source_directory(. source) -add_library (utils ${source}) -target_link_libraries(utils) +# ============================================================================== +# Utility Library +# ============================================================================== +set(utils_sources_c + logging.c + mymath.c + mystr.c + mysys.c +) +add_library(utils_lib ${utils_sources_c}) +# Set target-specific compiler flags +target_compile_options(utils_lib PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} +) +target_include_directories(utils_lib PRIVATE + ${GLib_INCLUDE_DIRS} + ${libCacheSim_include_dir} + ${libCacheSim_binary_include_dir} +) + +set(utils_sources ${utils_sources_c} PARENT_SCOPE) diff --git a/scripts/install_dependency.sh b/scripts/install_dependency.sh index ddeaf55f..396a9aa3 100755 --- a/scripts/install_dependency.sh +++ b/scripts/install_dependency.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -euo pipefail # Enable strict error checking @@ -10,47 +10,45 @@ NC='\033[0m' # No Color # Logging functions log_info() { - echo -e "${GREEN}[INFO]${NC} $1" + echo -e "${GREEN}[INFO]${NC} $1" } log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" + echo -e "${YELLOW}[WARN]${NC} $1" } log_error() { - echo -e "${RED}[ERROR]${NC} $1" >&2 + echo -e "${RED}[ERROR]${NC} $1" >&2 } # Cleanup function cleanup() { - local exit_code=$? - if [ $exit_code -ne 0 ]; then - log_error "Script failed with exit code $exit_code" - fi - cd "$CURR_DIR" - exit $exit_code + local exit_code=$? + if [[ ${exit_code} -ne 0 ]]; then + log_error "Script failed with exit code ${exit_code}" + fi + cd "${CURR_DIR}" + exit "${exit_code}" } trap cleanup EXIT # Help message show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Install dependencies for libCacheSim" - echo - echo "Options:" - echo " -h, --help Show this help message" - echo " -c, --cmake-only Install only CMake" - echo " -z, --zstd-only Install only Zstd" - echo " -x, --xgboost-only Install only XGBoost" - echo " -l, --lightgbm-only Install only LightGBM" - echo " -v, --verbose Enable verbose output" - echo - echo "If no options are specified, all dependencies will be installed" + echo "Usage: $0 [OPTIONS]" + echo "Install dependencies for libCacheSim" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -c, --cmake-only Install only CMake" + echo " -z, --zstd-only Install only Zstd" + echo " -x, --xgboost-only Install only XGBoost" + echo " -l, --lightgbm-only Install only LightGBM" + echo + echo "If no options are specified, all dependencies will be installed" } # Parse command line arguments -VERBOSE=false INSTALL_ALL=true INSTALL_CMAKE=false INSTALL_ZSTD=false @@ -58,207 +56,205 @@ INSTALL_XGBOOST=false INSTALL_LIGHTGBM=false while [[ $# -gt 0 ]]; do - case $1 in - -h|--help) - show_help - exit 0 - ;; - -c|--cmake-only) - INSTALL_ALL=false - INSTALL_CMAKE=true - shift - ;; - -z|--zstd-only) - INSTALL_ALL=false - INSTALL_ZSTD=true - shift - ;; - -x|--xgboost-only) - INSTALL_ALL=false - INSTALL_XGBOOST=true - shift - ;; - -l|--lightgbm-only) - INSTALL_ALL=false - INSTALL_LIGHTGBM=true - shift - ;; - -v|--verbose) - VERBOSE=true - shift - ;; - *) - log_error "Unknown option: $1" - show_help - exit 1 - ;; - esac + case $1 in + -h | --help) + show_help + exit 0 + ;; + -c | --cmake-only) + INSTALL_ALL=false + INSTALL_CMAKE=true + shift + ;; + -z | --zstd-only) + INSTALL_ALL=false + INSTALL_ZSTD=true + shift + ;; + -x | --xgboost-only) + INSTALL_ALL=false + INSTALL_XGBOOST=true + shift + ;; + -l | --lightgbm-only) + INSTALL_ALL=false + INSTALL_LIGHTGBM=true + shift + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac done # Store current directory CURR_DIR=$(pwd) # Check if running with sudo -if [ "$EUID" -eq 0 ]; then - log_warn "Please do not run this script as root/sudo" - exit 1 +if [[ ${EUID} -eq 0 ]]; then + log_warn "Please do not run this script as root/sudo" + exit 1 fi # OS detection and setup setup_ubuntu() { - log_info "Setting up Ubuntu dependencies..." - sudo apt update - sudo apt install -yqq build-essential google-perftools xxhash - sudo apt install -yqq libglib2.0-dev libunwind-dev - sudo apt install -yqq libgoogle-perftools-dev clang-tidy + log_info "Setting up Ubuntu dependencies..." + sudo apt update + sudo apt install -yqq build-essential google-perftools xxhash + sudo apt install -yqq libglib2.0-dev libunwind-dev + sudo apt install -yqq libgoogle-perftools-dev } setup_centos() { - log_info "Setting up CentOS dependencies..." - sudo yum install centos-release-scl - sudo yum install -y glib2-devel google-perftools-devel - # For the older llvm-toolset-7 (Clang 5.0.1) - sudo yum install llvm-toolset-7-clang-tools-extra - # Or for llvm-toolset-10 (Clang 10) - # sudo yum install llvm-toolset-10.0-clang-tools-extra - sudo scl enable llvm-toolset-10 bash + log_info "Setting up CentOS dependencies..." + sudo yum install centos-release-scl + sudo yum install -y glib2-devel google-perftools-devel + # For the older llvm-toolset-7 (Clang 5.0.1) + sudo yum install llvm-toolset-7-clang-tools-extra + # Or for llvm-toolset-10 (Clang 10) + # sudo yum install llvm-toolset-10.0-clang-tools-extra + sudo scl enable llvm-toolset-10 bash } setup_macOS() { - log_info "Setting up macOS dependencies..." - if ! command -v brew &> /dev/null; then - log_error "Homebrew is not installed. Please install Homebrew first." - exit 1 - fi - brew install glib google-perftools argp-standalone xxhash llvm wget cmake zstd xgboost lightgbm + log_info "Setting up macOS dependencies..." + if ! command -v brew &>/dev/null; then + log_error "Homebrew is not installed. Please install Homebrew first." + exit 1 + fi + brew install glib google-perftools argp-standalone xxhash llvm wget cmake zstd xgboost lightgbm } # Install CMake install_cmake() { - log_info "Installing CMake..." - local cmake_version="3.31.0" - local cmake_dir="$HOME/software/cmake" - - pushd /tmp/ > /dev/null - if [ ! -f "cmake-${cmake_version}-linux-x86_64.sh" ]; then - wget "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh" - fi - - mkdir -p "$cmake_dir" 2>/dev/null || true - bash "cmake-${cmake_version}-linux-x86_64.sh" --skip-license --prefix="$cmake_dir" - - # Add to shell config files if not already present - for shell_rc in "$HOME/.bashrc" "$HOME/.zshrc"; do - if [ -f "$shell_rc" ] && ! grep -q "PATH=\$HOME/software/cmake/bin:\$PATH" "$shell_rc"; then - echo "export PATH=\$HOME/software/cmake/bin:\$PATH" >> "$shell_rc" - fi - done - - # Source the updated PATH - export PATH="$cmake_dir/bin:$PATH" - popd > /dev/null + log_info "Installing CMake..." + local cmake_version="3.31.0" + local cmake_dir="${HOME}/software/cmake" + + pushd /tmp/ >/dev/null + if [[ ! -f "cmake-${cmake_version}-linux-x86_64.sh" ]]; then + wget "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh" + fi + + mkdir -p "${cmake_dir}" 2>/dev/null || true + bash "cmake-${cmake_version}-linux-x86_64.sh" --skip-license --prefix="${cmake_dir}" + + # Add to shell config files if not already present + for shell_rc in "${HOME}/.bashrc" "${HOME}/.zshrc"; do + # trunk-ignore(shellcheck/SC2016) + if [[ -f ${shell_rc} ]] && ! grep -q 'PATH=$HOME/software/cmake/bin:$PATH' "${shell_rc}"; then + # trunk-ignore(shellcheck/SC2016) + echo 'export PATH=$HOME/software/cmake/bin:$PATH' >>"${shell_rc}" + fi + done + + # Source the updated PATH + export PATH="${cmake_dir}/bin:${PATH}" + popd >/dev/null } # Install XGBoost install_xgboost() { - log_info "Installing XGBoost..." - pushd /tmp/ > /dev/null - if [ ! -d "xgboost" ]; then - git clone --recursive https://github.com/dmlc/xgboost - fi - pushd xgboost > /dev/null - mkdir -p build - pushd build > /dev/null - cmake .. - if [[ ${GITHUB_ACTIONS:-} == "true" ]]; then - make - else - make -j - fi - sudo make install - popd > /dev/null - popd > /dev/null - popd > /dev/null + log_info "Installing XGBoost..." + pushd /tmp/ >/dev/null + if [[ ! -d "xgboost" ]]; then + git clone --recursive https://github.com/dmlc/xgboost + fi + pushd xgboost >/dev/null + mkdir -p build + pushd build >/dev/null + cmake .. + if [[ ${GITHUB_ACTIONS-} == "true" ]]; then + make + else + make -j + fi + sudo make install + popd >/dev/null + popd >/dev/null + popd >/dev/null } # Install LightGBM install_lightgbm() { - log_info "Installing LightGBM..." - pushd /tmp/ > /dev/null - if [ ! -d "LightGBM" ]; then - git clone --recursive https://github.com/microsoft/LightGBM - fi - pushd LightGBM > /dev/null - mkdir -p build - pushd build > /dev/null - cmake .. - if [[ ${GITHUB_ACTIONS:-} == "true" ]]; then - make - else - make -j - fi - sudo make install - popd > /dev/null - popd > /dev/null - popd > /dev/null + log_info "Installing LightGBM..." + pushd /tmp/ >/dev/null + if [[ ! -d "LightGBM" ]]; then + git clone --recursive https://github.com/microsoft/LightGBM + fi + pushd LightGBM >/dev/null + mkdir -p build + pushd build >/dev/null + cmake .. + if [[ ${GITHUB_ACTIONS-} == "true" ]]; then + make + else + make -j + fi + sudo make install + popd >/dev/null + popd >/dev/null + popd >/dev/null } # Install Zstd install_zstd() { - log_info "Installing Zstd..." - pushd /tmp/ > /dev/null - local zstd_version="1.5.0" - if [ ! -f "zstd-${zstd_version}.tar.gz" ]; then - wget "https://github.com/facebook/zstd/releases/download/v${zstd_version}/zstd-${zstd_version}.tar.gz" - tar xf "zstd-${zstd_version}.tar.gz" - fi - pushd "zstd-${zstd_version}/build/cmake/" > /dev/null - mkdir -p _build - pushd _build > /dev/null - cmake .. - make -j - sudo make install - popd > /dev/null - popd > /dev/null - popd > /dev/null + log_info "Installing Zstd..." + pushd /tmp/ >/dev/null + local zstd_version="1.5.0" + if [[ ! -f "zstd-${zstd_version}.tar.gz" ]]; then + wget "https://github.com/facebook/zstd/releases/download/v${zstd_version}/zstd-${zstd_version}.tar.gz" + tar xf "zstd-${zstd_version}.tar.gz" + fi + pushd "zstd-${zstd_version}/build/cmake/" >/dev/null + mkdir -p _build + pushd _build >/dev/null + cmake .. + make -j + sudo make install + popd >/dev/null + popd >/dev/null + popd >/dev/null } # Main installation logic main() { - # Detect OS and setup basic dependencies - if [ -n "$(uname -a | grep -E "Ubuntu|Debian|WSL")" ]; then - setup_ubuntu - elif [ -n "$(uname -a | grep Darwin)" ]; then - setup_macOS - else - setup_centos - fi + # Detect OS and setup basic dependencies + if uname -a | grep -qE "Ubuntu|Debian|WSL"; then + setup_ubuntu + elif uname -a | grep -q Darwin; then + setup_macOS + else + setup_centos + fi - # Install requested components only on non macOS computers - # Libraries already installed on macOS using brew in setup_macOS - if [[ -z "$(uname -a | grep Darwin)" ]]; then - if [ "$INSTALL_ALL" = true ] || [ "$INSTALL_CMAKE" = true ]; then - install_cmake - fi + # Install requested components only on non macOS computers + # Libraries already installed on macOS using brew in setup_macOS + if ! uname -a | grep -q Darwin; then + if [[ ${INSTALL_ALL} == true ]] || [[ ${INSTALL_CMAKE} == true ]]; then + install_cmake + fi - if [ "$INSTALL_ALL" = true ] || [ "$INSTALL_ZSTD" = true ]; then - install_zstd - fi + if [[ ${INSTALL_ALL} == true ]] || [[ ${INSTALL_ZSTD} == true ]]; then + install_zstd + fi - if [ "$INSTALL_ALL" = true ] || [ "$INSTALL_XGBOOST" = true ]; then - if [[ ! ${GITHUB_ACTIONS:-} == "true" ]]; then - install_xgboost - fi - fi + if [[ ${INSTALL_ALL} == true ]] || [[ ${INSTALL_XGBOOST} == true ]]; then + if [[ ${GITHUB_ACTIONS-} != "true" ]]; then + install_xgboost + fi + fi - if [ "$INSTALL_ALL" = true ] || [ "$INSTALL_LIGHTGBM" = true ]; then - if [[ ! ${GITHUB_ACTIONS:-} == "true" ]]; then - install_lightgbm - fi - fi - fi + if [[ ${INSTALL_ALL} == true ]] || [[ ${INSTALL_LIGHTGBM} == true ]]; then + if [[ ${GITHUB_ACTIONS-} != "true" ]]; then + install_lightgbm + fi + fi + fi - log_info "Installation completed successfully!" + log_info "Installation completed successfully!" } # Run main function diff --git a/scripts/install_dev_dependency.sh b/scripts/install_dev_dependency.sh new file mode 100755 index 00000000..0507ff58 --- /dev/null +++ b/scripts/install_dev_dependency.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -euo pipefail # Enable strict error checking + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Cleanup function +cleanup() { + local exit_code=$? + if [[ ${exit_code} -ne 0 ]]; then + log_error "Script failed with exit code ${exit_code}" + fi + cd "${CURR_DIR}" + exit "${exit_code}" +} + +trap cleanup EXIT + +# Store current directory +CURR_DIR=$(pwd) + +# Main installation logic +main() { + # Detect OS and setup basic dependencies + if [[ -f /etc/os-release ]] && grep -qiE 'ubuntu|debian' /etc/os-release; then + sudo apt install -yqq gdb valgrind clang-tidy clang-format + # trunk-ignore(shellcheck/SC2312) + elif [[ $(uname -a) == "Darwin" ]]; then + brew install gdb valgrind clang-format + elif grep -qi 'microsoft' /proc/version 2>/dev/null; then + # WSL detection + sudo apt install -yqq gdb valgrind clang-tidy clang-format + else + log_error "Unsupported operating system. Only Ubuntu, Debian, WSL, and macOS are supported. Some dependencies may not be installed." + fi + + bash ${CURR_DIR}/setup-hooks.sh + + log_info "Installation completed successfully!" +} + +# Run main function +main diff --git a/scripts/lint.sh b/scripts/lint.sh deleted file mode 100755 index 2cabec32..00000000 --- a/scripts/lint.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Check if clang-tidy is installed -if ! command -v clang-tidy &> /dev/null; then - echo -e "${RED}Error: clang-tidy is not installed${NC}" - echo "Please install clang-tidy:" - echo " Ubuntu/Debian: sudo apt install clang-tidy" - echo " macOS: brew install llvm" - exit 1 -fi - -# Function to print usage -usage() { - echo "Usage: $0 [options]" - echo "Options:" - echo " -h, --help Show this help message" - echo " -f, --fix Apply suggested fixes" - echo " -p, --path PATH Path to compile_commands.json" - echo " --all Check all files (default: only changed files)" - exit 1 -} - -# Parse arguments -FIX=0 -CHECK_ALL=0 -COMPILE_COMMANDS_DIR="_build" - -while [[ $# -gt 0 ]]; do - case $1 in - -h|--help) - usage - ;; - -f|--fix) - FIX=1 - shift - ;; - -p|--path) - COMPILE_COMMANDS_DIR="$2" - shift 2 - ;; - --all) - CHECK_ALL=1 - shift - ;; - *) - echo -e "${RED}Unknown option: $1${NC}" - usage - ;; - esac -done - -# Ensure we're in the project root -cd "${PROJECT_ROOT}" - -# Check if compile_commands.json exists -if [ ! -f "${COMPILE_COMMANDS_DIR}/compile_commands.json" ]; then - echo -e "${YELLOW}Warning: compile_commands.json not found in ${COMPILE_COMMANDS_DIR}${NC}" - echo "Building project to generate compile_commands.json..." - - # Create build directory if it doesn't exist - mkdir -p "${COMPILE_COMMANDS_DIR}" - - # Generate compile_commands.json - pushd "${COMPILE_COMMANDS_DIR}" > /dev/null - cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. - popd > /dev/null -fi - -# Get list of files to check -if [ $CHECK_ALL -eq 1 ]; then - # Find all C/C++ source files - FILES=$(find libCacheSim -type f \( -name "*.c" -o -name "*.cpp" -o -name "*.cc" -o -name "*.h" -o -name "*.hpp" \)) -else - # Get only changed files - FILES=$(git diff --name-only HEAD | grep -E "\.(c|cpp|cc|h|hpp)$" || true) - if [ -z "$FILES" ]; then - echo -e "${YELLOW}No C/C++ files changed${NC}" - exit 0 - fi -fi - -# Run clang-tidy -echo -e "${GREEN}Running clang-tidy...${NC}" -ERRORS=0 -for file in $FILES; do - echo "Checking $file..." - if [ $FIX -eq 1 ]; then - # Run with fix and create backup files - clang-tidy -p "${COMPILE_COMMANDS_DIR}" --fix "$file" || ERRORS=$? - else - clang-tidy -p "${COMPILE_COMMANDS_DIR}" "$file" || ERRORS=$? - fi -done - -if [ $ERRORS -eq 0 ]; then - echo -e "${GREEN}No linting errors found${NC}" -else - echo -e "${RED}Linting errors found${NC}" - exit 1 -fi \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ec4aa79f..c4e55be3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,42 +1,48 @@ -set(coreLib ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) - -message(STATUS "coreLib: ${coreLib}") - -add_executable(testReader test_traceReader.c) -target_link_libraries(testReader ${coreLib}) - -add_executable(testDistUtils test_dist.c) -target_link_libraries(testDistUtils ${coreLib}) - -add_executable(testProfilerLRU test_profilerLRU.c) -target_link_libraries(testProfilerLRU ${coreLib}) - -add_executable(testSimulator test_simulator.c) -target_link_libraries(testSimulator ${coreLib}) - -add_executable(testEvictionAlgo test_evictionAlgo.c) -target_link_libraries(testEvictionAlgo ${coreLib}) - -add_executable(testMRC test_MRC.c) -target_link_libraries(testMRC ${coreLib}) - -add_executable(testAdmissionAlgo test_admissionAlgo.c) -target_link_libraries(testAdmissionAlgo ${coreLib}) - - -add_executable(testPrefetchAlgo test_prefetchAlgo.c) -target_link_libraries(testPrefetchAlgo ${coreLib}) - -add_executable(testDataStructure test_dataStructure.c) -target_link_libraries(testDataStructure ${coreLib}) - -add_executable(testUtils test_utils.c) -target_link_libraries(testUtils ${coreLib}) - -add_executable(testMrcProfiler test_mrcProfiler.cpp) -target_link_libraries(testMrcProfiler mrcProfilerLib m zstd dl pthread -Wl,--whole-archive libCacheSim -Wl,--no-whole-archive ${coreLib}) -target_link_options(testMrcProfiler PRIVATE "-Wl,--export-dynamic") +# Helper function to create test executables with explicit compiler flags +function(add_test_executable target_name source_file) + add_executable(${target_name} ${source_file}) + + # Apply appropriate compiler flags based on source file language + get_filename_component(file_ext ${source_file} EXT) + if(file_ext STREQUAL ".cpp" OR file_ext STREQUAL ".cxx" OR file_ext STREQUAL ".cc") + # C++ source file + target_compile_options(${target_name} PRIVATE + ${LIBCACHESIM_CXX_FLAGS} + ${tcmalloc_flags} + ) + set_target_properties(${target_name} PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO + ) + else() + # C source file + target_compile_options(${target_name} PRIVATE + ${LIBCACHESIM_C_FLAGS} + ${tcmalloc_flags} + ) + endif() + + target_link_libraries(${target_name} ${all_modules} ${dependency_libs}) +endfunction() + +# Create C test executables +add_test_executable(testReader test_traceReader.c) +add_test_executable(testDistUtils test_dist.c) +add_test_executable(testProfilerLRU test_profilerLRU.c) +add_test_executable(testSimulator test_simulator.c) +add_test_executable(testEvictionAlgo test_evictionAlgo.c) +add_test_executable(testMRC test_MRC.c) +add_test_executable(testAdmissionAlgo test_admissionAlgo.c) +add_test_executable(testPrefetchAlgo test_prefetchAlgo.c) +add_test_executable(testDataStructure test_dataStructure.c) +add_test_executable(testUtils test_utils.c) + +# Create C++ test executable (mrcProfiler test) +# add_test_executable(testMrcProfiler test_mrcProfiler.cpp) +# target_link_libraries(testMrcProfiler mrcProfiler_lib) +# target_link_options(testMrcProfiler PRIVATE "-Wl,--export-dynamic") @@ -49,16 +55,16 @@ add_test(NAME testAdmissionAlgo COMMAND testAdmissionAlgo WORKING_DIRECTORY .) add_test(NAME testPrefetchAlgo COMMAND testPrefetchAlgo WORKING_DIRECTORY .) add_test(NAME testDataStructure COMMAND testDataStructure WORKING_DIRECTORY .) add_test(NAME testUtils COMMAND testUtils WORKING_DIRECTORY .) -add_test(NAME testMrcProfiler COMMAND testMrcProfiler WORKING_DIRECTORY .) +# add_test(NAME testMrcProfiler COMMAND testMrcProfiler WORKING_DIRECTORY .) # if (ENABLE_GLCACHE) # add_executable(testGLCache test_glcache.c) -# target_link_libraries(testGLCache ${coreLib}) +# target_link_libraries(testGLCache ${all_modules} ${dependency_libs}) # add_test(NAME testGLCache COMMAND testGLCache WORKING_DIRECTORY .) # endif (ENABLE_GLCACHE) if (ENABLE_3L_CACHE) add_executable(test3LCache test_3lcache.c) - target_link_libraries(test3LCache ${coreLib}) + target_link_libraries(test3LCache ${all_modules} ${dependency_libs}) add_test(NAME test3LCache COMMAND test3LCache WORKING_DIRECTORY .) endif (ENABLE_3L_CACHE) \ No newline at end of file