Skip to content

Commit 77c15bb

Browse files
haochengxiahaochengxia
committed
Fix CI for python binding
1 parent c37c6de commit 77c15bb

6 files changed

Lines changed: 86 additions & 84 deletions

File tree

‎libCacheSim-python/CMakeLists.txt‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
1010
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
1111
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
1212

13+
option(ENABLE_LRB "enable LRB" OFF)
14+
option(ENABLE_3L_CACHE "enable 3LCache" OFF)
15+
1316
project(
1417
${SKBUILD_PROJECT_NAME}
1518
VERSION ${SKBUILD_PROJECT_VERSION}
@@ -46,13 +49,6 @@ else()
4649
add_compile_definitions(LOGLEVEL=7)
4750
endif()
4851

49-
option(ENABLE_LRB "enable LRB" ON)
50-
option(ENABLE_GLCACHE "enable GLCache" ON)
51-
option(ENABLE_3L_CACHE "enable 3LCache" ON)
52-
add_compile_definitions(ENABLE_GLCACHE=1)
53-
add_compile_definitions(ENABLE_LRB=1)
54-
add_compile_definitions(ENABLE_3L_CACHE=1)
55-
5652
# find python and pybind11
5753
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
5854
find_package(pybind11 CONFIG REQUIRED)
@@ -79,11 +75,15 @@ set_target_properties(utils PROPERTIES
7975
target_compile_options(utils PRIVATE -fPIC)
8076
target_link_libraries(utils PRIVATE ${GLIB2_LIBRARIES})
8177

82-
file(GLOB_RECURSE CACHE_SOURCES
78+
file(GLOB CACHE_SOURCES
8379
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/*.c"
8480
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/*.c"
8581
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/admission/*.c"
8682
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/prefetch/*.c"
83+
84+
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/fifo/*.c"
85+
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/other/*.c"
86+
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/belady/*.c"
8787
)
8888

8989
# Try to find XGBoost and LightGBM (optional)
@@ -123,19 +123,17 @@ if(ENABLE_LIGHTGBM)
123123
else()
124124
set(LRB_SOURCES "")
125125
set(3L_CACHE_SOURCES "")
126-
127-
# disable the features
128-
add_compile_definitions(ENABLE_3L_CACHE=0)
129-
add_compile_definitions(ENABLE_LRB=0)
130126
endif()
131127

132-
file(GLOB_RECURSE CACHE_CPP_SOURCES
128+
file(GLOB CACHE_CPP_SOURCES
133129
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/LHD/*"
134130
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/cache/eviction/cpp/*"
135131
)
136132

137-
file(GLOB_RECURSE DS_SOURCES
133+
file(GLOB DS_SOURCES
138134
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/dataStructure/*.c"
135+
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/dataStructure/hashtable/*.c"
136+
"${CMAKE_CURRENT_SOURCE_DIR}/../libCacheSim/dataStructure/hash/murmur3.c"
139137
)
140138

141139
file(GLOB_RECURSE READER_SOURCES

‎libCacheSim-python/libcachesim/eviction.py‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,8 @@ def init_cache(self, cache_size: int, **kwargs):
108108
ain_size_ratio = kwargs.get('ain_size_ratio', 0.25)
109109
aout_size_ratio = kwargs.get('aout_size_ratio', 0.5)
110110

111-
if ain_size_ratio <= 0 or ain_size_ratio >= 1:
112-
msg = "ain_size_ratio must be between 0 and 1"
113-
raise ValueError(msg)
114-
if aout_size_ratio <= 0:
115-
msg = "aout_size_ratio must be greater than 0"
111+
if ain_size_ratio <= 0 or aout_size_ratio <= 0:
112+
msg = "ain_size_ratio and aout_size_ratio must be greater than 0"
116113
raise ValueError(msg)
117114

118115
self.ain_size_ratio = ain_size_ratio
@@ -210,11 +207,8 @@ def init_cache(self, cache_size: int, **kwargs):
210207
ghost_size_ratio = kwargs.get('ghost_size_ratio', 0.9)
211208
move_to_main_threshold = kwargs.get('move_to_main_threshold', 2)
212209

213-
if fifo_size_ratio <= 0 or fifo_size_ratio >= 1:
214-
msg = "fifo_size_ratio must be between 0 and 1"
215-
raise ValueError(msg)
216-
if ghost_size_ratio <= 0:
217-
msg = "ghost_size_ratio must be greater than 0"
210+
if fifo_size_ratio <= 0 or ghost_size_ratio <= 0:
211+
msg = "fifo_size_ratio and ghost_size_ratio must be greater than 0"
218212
raise ValueError(msg)
219213
if move_to_main_threshold < 0:
220214
msg = "move_to_main_threshold must be greater or equal to 0"
@@ -286,8 +280,8 @@ def init_cache(self, cache_size: int, **kwargs):
286280
main_cache = kwargs.get('main_cache', "SLRU")
287281
window_size = kwargs.get('window_size', 0.01)
288282

289-
if window_size <= 0 or window_size >= 1:
290-
msg = "window_size must be between 0 and 1"
283+
if window_size <= 0:
284+
msg = "window_size must be greater than 0"
291285
raise ValueError(msg)
292286

293287
self.main_cache = main_cache

‎libCacheSim-python/src/cache_init.hpp‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -69,43 +69,50 @@ static inline cache_t *create_sim_cache(const char *eviction_algo,
6969
} else {
7070
const char *window_size = strstr(eviction_params, "window-size=");
7171
if (window_size == NULL) {
72-
char *new_params = (char *)malloc(strlen(eviction_params) + 20);
73-
if (strlen(eviction_params) > 0) {
74-
sprintf(new_params, "%s,window-size=0.01", eviction_params);
72+
char dest[30];
73+
strncpy(dest, eviction_params, sizeof(dest) - 1);
74+
dest[sizeof(dest) - 1] = '\0';
75+
size_t param_len = strlen(dest);
76+
char *new_params = (char *)malloc(param_len + 20);
77+
if (param_len > 0) {
78+
snprintf(new_params, param_len + 20, "%s,window-size=0.01",
79+
eviction_params);
7580
} else { // if eviction_params is "", no comma is needed
76-
sprintf(new_params, "window-size=0.01");
81+
snprintf(new_params, 20, "window-size=0.01");
7782
}
7883
cache = WTinyLFU_init(cc_params, new_params);
84+
free(new_params);
7985
} else {
8086
cache = WTinyLFU_init(cc_params, eviction_params);
8187
}
8288
}
8389
} else if (strcasecmp(eviction_algo, "wtinyLFU") == 0) {
8490
cache = WTinyLFU_init(cc_params, eviction_params);
85-
/* TODO(haocheng): add belady support, since we remove the trace path, format info needs to be passed in through another parameter */
86-
// } else if (strcasecmp(eviction_algo, "belady") == 0 &&
87-
// strcasestr(trace_path, "lcs") == NULL) {
88-
// if (strcasestr(trace_path, "oracleGeneral") == NULL) {
89-
// WARN("belady is only supported for oracleGeneral and lcs trace\n");
90-
// WARN("to convert a trace to lcs format\n");
91-
// WARN("./bin/traceConv input_trace trace_format output_trace\n");
92-
// WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt\n");
93-
// exit(1);
94-
// }
95-
// cache = Belady_init(cc_params, eviction_params);
91+
/* TODO(haocheng): add belady support, since we remove the trace path,
92+
* format info needs to be passed in through another parameter */
93+
// } else if (strcasecmp(eviction_algo, "belady") == 0 &&
94+
// strcasestr(trace_path, "lcs") == NULL) {
95+
// if (strcasestr(trace_path, "oracleGeneral") == NULL) {
96+
// WARN("belady is only supported for oracleGeneral and lcs trace\n");
97+
// WARN("to convert a trace to lcs format\n");
98+
// WARN("./bin/traceConv input_trace trace_format output_trace\n");
99+
// WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt\n");
100+
// exit(1);
101+
// }
102+
// cache = Belady_init(cc_params, eviction_params);
96103
} else if (strcasecmp(eviction_algo, "nop") == 0) {
97104
cache = nop_init(cc_params, eviction_params);
98-
// } else if (strcasecmp(eviction_algo, "beladySize") == 0) {
99-
// if (strcasestr(trace_path, "oracleGeneral") == NULL &&
100-
// strcasestr(trace_path, "lcs") == NULL) {
101-
// WARN("beladySize is only supported for oracleGeneral and lcs trace\n");
102-
// WARN("to convert a trace to lcs format\n");
103-
// WARN("./bin/traceConv input_trace trace_format output_trace\n");
104-
// WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt\n");
105-
// exit(1);
106-
// }
107-
// cc_params.hashpower = MAX(cc_params.hashpower - 8, 16);
108-
// cache = BeladySize_init(cc_params, eviction_params);
105+
// } else if (strcasecmp(eviction_algo, "beladySize") == 0) {
106+
// if (strcasestr(trace_path, "oracleGeneral") == NULL &&
107+
// strcasestr(trace_path, "lcs") == NULL) {
108+
// WARN("beladySize is only supported for oracleGeneral and lcs
109+
// trace\n"); WARN("to convert a trace to lcs format\n");
110+
// WARN("./bin/traceConv input_trace trace_format output_trace\n");
111+
// WARN("./bin/traceConv ../data/cloudPhysicsIO.txt txt\n");
112+
// exit(1);
113+
// }
114+
// cc_params.hashpower = MAX(cc_params.hashpower - 8, 16);
115+
// cache = BeladySize_init(cc_params, eviction_params);
109116
} else if (strcasecmp(eviction_algo, "fifo-reinsertion") == 0 ||
110117
strcasecmp(eviction_algo, "clock") == 0 ||
111118
strcasecmp(eviction_algo, "second-chance") == 0) {

‎libCacheSim-python/src/pylibcachesim.cpp‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
#include <iostream>
44

55
#include "cache_init.hpp"
6-
#include "utils.hpp"
76
#include "libCacheSim.h"
7+
#include "utils.hpp"
88

99
#define STRINGIFY(x) #x
1010
#define MACRO_STRINGIFY(x) STRINGIFY(x)
@@ -66,10 +66,9 @@ PYBIND11_MODULE(_libcachesim, m) {
6666
/**
6767
* @brief Request structure
6868
*/
69-
py::class_<request_t, std::unique_ptr<request_t, RequestDeleter>>(m, "Request")
70-
.def(py::init([]() {
71-
return new_request();
72-
}))
69+
py::class_<request_t, std::unique_ptr<request_t, RequestDeleter>>(m,
70+
"Request")
71+
.def(py::init([]() { return new_request(); }))
7372
.def_readwrite("clock_time", &request_t::clock_time)
7473
.def_readwrite("hv", &request_t::hv)
7574
.def_readwrite("obj_id", &request_t::obj_id)
@@ -85,13 +84,15 @@ PYBIND11_MODULE(_libcachesim, m) {
8584
.def_readwrite("trace_path", &reader_t::trace_path)
8685
.def_readwrite("file_size", &reader_t::file_size)
8786
// methods
88-
.def("get_wss", [](reader_t& self, bool ignore_obj_size) {
89-
90-
int64_t wss_obj = 0, wss_byte = 0;
91-
cal_working_set_size(&self, &wss_obj, &wss_byte);
92-
return ignore_obj_size ? wss_obj : wss_byte;
93-
}, py::arg("ignore_obj_size") = false,
94-
R"pbdoc(
87+
.def(
88+
"get_wss",
89+
[](reader_t& self, bool ignore_obj_size) {
90+
int64_t wss_obj = 0, wss_byte = 0;
91+
cal_working_set_size(&self, &wss_obj, &wss_byte);
92+
return ignore_obj_size ? wss_obj : wss_byte;
93+
},
94+
py::arg("ignore_obj_size") = false,
95+
R"pbdoc(
9596
Get the working set size of the trace.
9697
9798
Args:
@@ -170,12 +171,11 @@ PYBIND11_MODULE(_libcachesim, m) {
170171
*/
171172
m.def(
172173
"create_cache",
173-
[](const std::string& eviction_algo,
174-
const uint64_t cache_size, const std::string& eviction_params,
175-
bool consider_obj_metadata) {
176-
cache_t* ptr = create_sim_cache(
177-
eviction_algo.c_str(), cache_size,
178-
eviction_params.c_str(), consider_obj_metadata);
174+
[](const std::string& eviction_algo, const uint64_t cache_size,
175+
const std::string& eviction_params, bool consider_obj_metadata) {
176+
cache_t* ptr =
177+
create_sim_cache(eviction_algo.c_str(), cache_size,
178+
eviction_params.c_str(), consider_obj_metadata);
179179
return std::unique_ptr<cache_t, CacheDeleter>(ptr);
180180
},
181181
py::arg("eviction_algo"), py::arg("cache_size"),
@@ -193,8 +193,9 @@ PYBIND11_MODULE(_libcachesim, m) {
193193
Cache: A new cache instance.
194194
)pbdoc");
195195

196-
/* TODO(haocheng): should we support all parameters in the common_cache_params_t? (hash_power, etc.) */
197-
196+
/* TODO(haocheng): should we support all parameters in the
197+
* common_cache_params_t? (hash_power, etc.) */
198+
198199
// Currently supported eviction algorithms with direct initialization:
199200
// - "ARC"
200201
// - "Clock"
@@ -300,7 +301,8 @@ PYBIND11_MODULE(_libcachesim, m) {
300301
Cache: A new LRB cache instance.
301302
)pbdoc");
302303
#else
303-
// TODO(haocheng): add a dummy function to avoid the error when LRB is not enabled
304+
// TODO(haocheng): add a dummy function to avoid the error when LRB is not
305+
// enabled
304306
m.def(
305307
"LRB_init",
306308
[](uint64_t cache_size, std::string objective) {
@@ -406,7 +408,8 @@ PYBIND11_MODULE(_libcachesim, m) {
406408
Cache: A new ThreeL cache instance.
407409
)pbdoc");
408410
#else
409-
// TODO(haocheng): add a dummy function to avoid the error when ThreeLCache is not enabled
411+
// TODO(haocheng): add a dummy function to avoid the error when ThreeLCache is
412+
// not enabled
410413
m.def(
411414
"ThreeLCache_init",
412415
[](uint64_t cache_size, std::string objective) {

‎libCacheSim-python/src/utils.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include "libCacheSim/reader.h"
55

66
static inline void cal_working_set_size(reader_t *reader, int64_t *wss_obj,
7-
int64_t *wss_byte) {
7+
int64_t *wss_byte) {
88
reset_reader(reader);
99
request_t *req = new_request();
1010
GHashTable *obj_table = g_hash_table_new(g_direct_hash, g_direct_equal);

‎libCacheSim-python/tests/test_eviction.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
import pytest
2-
import os
32

4-
from libcachesim import Cache, Reader, Request, create_cache, open_trace, TraceType
53
from libcachesim import (
6-
FIFO, ARC, Clock, LRB, LRU, S3FIFO, Sieve, ThreeLCache,
7-
TinyLFU, TwoQ,
4+
ARC,
5+
FIFO,
6+
LRU,
7+
S3FIFO,
8+
Clock,
9+
Sieve,
10+
TinyLFU,
11+
TwoQ,
12+
create_cache,
813
)
9-
1014
from tests.utils import get_reference_data
1115

1216

1317
@pytest.mark.parametrize("eviction_algo", [
1418
FIFO,
1519
ARC,
1620
Clock,
17-
# LRB,
1821
LRU,
1922
S3FIFO,
2023
Sieve,
21-
# ThreeLCache,
2224
TinyLFU,
2325
TwoQ,
2426
])
@@ -45,11 +47,9 @@ def test_eviction_algo(eviction_algo, cache_size_ratio, mock_reader):
4547
"FIFO",
4648
"ARC",
4749
"Clock",
48-
# "LRB",
4950
"LRU",
5051
"S3FIFO",
5152
"Sieve",
52-
# "3LCache",
5353
"TinyLFU",
5454
"TwoQ",
5555
])

0 commit comments

Comments
 (0)