Skip to content

Commit af4073d

Browse files
1a1a11a1a1a11a
committed
Refactor traceAnalyzer to improve output handling and add popularity decay option
1 parent 70ce858 commit af4073d

File tree

5 files changed

+135
-75
lines changed

5 files changed

+135
-75
lines changed

‎libCacheSim/bin/traceAnalyzer/cli_parser.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,13 @@ static struct argp_option options[] = {
7474
"enable popularity analysis, output freq:cnt in dataname.popularity file, "
7575
"and prints the skewness to stdout and stat file",
7676
3},
77-
{"popularityDacay", OPTION_ENABLE_POPULARITY_DECAY, NULL,
77+
{"popularityDecay", OPTION_ENABLE_POPULARITY_DECAY, NULL,
7878
OPTION_ARG_OPTIONAL,
7979
"enable popularity decay analysis, this calculates popularity fade as a "
8080
"heatmap. It is an expensive analysis, enable only when needed.",
8181
3},
82+
{"popularityDacay", OPTION_ENABLE_POPULARITY_DECAY, NULL,
83+
OPTION_ARG_OPTIONAL | OPTION_HIDDEN, NULL, 3},
8284
{"reuse", OPTION_ENABLE_REUSE, NULL, OPTION_ARG_OPTIONAL,
8385
"reuse analysis, output a reuse distribution (both real time and virtual "
8486
"time) in dataname.reuse file",
@@ -110,8 +112,8 @@ static struct argp_option options[] = {
110112

111113
{NULL, 0, NULL, 0, "common parameters:", 0},
112114

113-
{"output", OPTION_OUTPUT_PATH, "", OPTION_ARG_OPTIONAL, "Output path", 8},
114-
{"verbose", OPTION_VERBOSE, NULL, OPTION_ARG_OPTIONAL,
115+
{"output", OPTION_OUTPUT_PATH, "PATH", 0, "Output path prefix", 8},
116+
{"verbose", OPTION_VERBOSE, NULL, 0,
115117
"Produce verbose output", 8},
116118
{NULL, 0, NULL, 0, NULL, 0}};
117119

@@ -128,6 +130,10 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
128130
arguments->trace_type_params = arg;
129131
break;
130132
case OPTION_OUTPUT_PATH:
133+
if (arg == nullptr) {
134+
argp_usage(state);
135+
exit(1);
136+
}
131137
strncpy(arguments->ofilepath, arg, OFILEPATH_LEN - 1);
132138
arguments->ofilepath[OFILEPATH_LEN - 1] = '\0';
133139
break;
@@ -149,6 +155,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
149155
case OPTION_TRACK_N_HIT:
150156
arguments->analysis_param.track_n_hit = atoi(arg);
151157
break;
158+
case OPTION_TRACK_N_POPULAR:
159+
arguments->analysis_param.track_n_popular = atoi(arg);
160+
break;
152161
case OPTION_ENABLE_ALL:
153162
arguments->analysis_option.req_rate = true;
154163
arguments->analysis_option.access_pattern = true;
@@ -187,7 +196,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
187196
break;
188197

189198
case OPTION_VERBOSE:
190-
arguments->verbose = is_true(arg) ? true : false;
199+
arguments->verbose = true;
191200
break;
192201
case ARGP_KEY_ARG:
193202
if (state->arg_num >= N_ARGS) {
@@ -236,7 +245,6 @@ static void init_arg(struct arguments *args) {
236245

237246
args->trace_path = NULL;
238247
args->trace_type_params = NULL;
239-
args->verbose = true;
240248
memset(args->ofilepath, 0, OFILEPATH_LEN);
241249
args->n_req = -1;
242250
args->verbose = false;

‎libCacheSim/bin/traceAnalyzer/main.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ int main(int argc, char *argv[]) {
1616
args.reader, args.ofilepath, args.analysis_option, args.analysis_param);
1717
stat->run();
1818

19-
ofstream ofs("traceStat", ios::out | ios::app);
19+
ofstream ofs(args.ofilepath + string(".traceStat"), ios::out | ios::app);
2020
ofs << *stat << endl;
2121
ofs.close();
2222
cout << *stat;

‎libCacheSim/traceAnalyzer/analyzer.cpp‎

Lines changed: 94 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,76 @@ void traceAnalyzer::TraceAnalyzer::cleanup() {
8989
void traceAnalyzer::TraceAnalyzer::run() {
9090
if (has_run_) return;
9191

92+
auto dump_outputs = [&]() {
93+
ofstream ofs(output_path_ + ".stat", ios::out | ios::app);
94+
ofs << gen_stat_str() << endl;
95+
ofs.close();
96+
97+
if (ttl_stat_ != nullptr) {
98+
ttl_stat_->dump(output_path_);
99+
}
100+
101+
if (req_rate_stat_ != nullptr) {
102+
req_rate_stat_->dump(output_path_);
103+
}
104+
105+
if (reuse_stat_ != nullptr) {
106+
reuse_stat_->dump(output_path_);
107+
}
108+
109+
if (size_stat_ != nullptr) {
110+
size_stat_->dump(output_path_);
111+
}
112+
113+
if (access_stat_ != nullptr) {
114+
access_stat_->dump(output_path_);
115+
}
116+
117+
if (popularity_stat_ != nullptr) {
118+
popularity_stat_->dump(output_path_);
119+
}
120+
121+
if (popularity_decay_stat_ != nullptr) {
122+
popularity_decay_stat_->dump(output_path_);
123+
}
124+
125+
if (prob_at_age_ != nullptr) {
126+
prob_at_age_->dump(output_path_);
127+
}
128+
129+
if (lifetime_stat_ != nullptr) {
130+
lifetime_stat_->dump(output_path_);
131+
}
132+
133+
if (create_future_reuse_ != nullptr) {
134+
create_future_reuse_->dump(output_path_);
135+
}
136+
137+
// if (write_reuse_stat_ != nullptr) {
138+
// write_reuse_stat_->dump(output_path_);
139+
// }
140+
141+
// if (write_future_reuse_stat_ != nullptr) {
142+
// write_future_reuse_stat_->dump(output_path_);
143+
// }
144+
145+
if (scan_detector_ != nullptr) {
146+
scan_detector_->dump(output_path_);
147+
}
148+
149+
has_run_ = true;
150+
};
151+
92152
request_t *req = new_request();
93153
read_one_req(reader_, req);
154+
if (!req->valid) {
155+
start_ts_ = 0;
156+
end_ts_ = 0;
157+
post_processing();
158+
free_request(req);
159+
dump_outputs();
160+
return;
161+
}
94162
start_ts_ = req->clock_time;
95163
int32_t curr_time_window_idx = 0;
96164
int next_time_window_ts = time_window_;
@@ -226,67 +294,35 @@ void traceAnalyzer::TraceAnalyzer::run() {
226294

227295
free_request(req);
228296

229-
ofstream ofs("stat", ios::out | ios::app);
230-
ofs << gen_stat_str() << endl;
231-
ofs.close();
232-
233-
if (ttl_stat_ != nullptr) {
234-
ttl_stat_->dump(output_path_);
235-
}
236-
237-
if (req_rate_stat_ != nullptr) {
238-
req_rate_stat_->dump(output_path_);
239-
}
240-
241-
if (reuse_stat_ != nullptr) {
242-
reuse_stat_->dump(output_path_);
243-
}
244-
245-
if (size_stat_ != nullptr) {
246-
size_stat_->dump(output_path_);
247-
}
248-
249-
if (access_stat_ != nullptr) {
250-
access_stat_->dump(output_path_);
251-
}
252-
253-
if (popularity_stat_ != nullptr) {
254-
popularity_stat_->dump(output_path_);
255-
}
256-
257-
if (popularity_decay_stat_ != nullptr) {
258-
popularity_decay_stat_->dump(output_path_);
259-
}
260-
261-
if (prob_at_age_ != nullptr) {
262-
prob_at_age_->dump(output_path_);
263-
}
264-
265-
if (lifetime_stat_ != nullptr) {
266-
lifetime_stat_->dump(output_path_);
267-
}
268-
269-
if (create_future_reuse_ != nullptr) {
270-
create_future_reuse_->dump(output_path_);
271-
}
272-
273-
// if (write_reuse_stat_ != nullptr) {
274-
// write_reuse_stat_->dump(output_path_);
275-
// }
276-
277-
// if (write_future_reuse_stat_ != nullptr) {
278-
// write_future_reuse_stat_->dump(output_path_);
279-
// }
280-
281-
if (scan_detector_ != nullptr) {
282-
scan_detector_->dump(output_path_);
283-
}
284-
285-
has_run_ = true;
297+
dump_outputs();
286298
}
287299

288300
string traceAnalyzer::TraceAnalyzer::gen_stat_str() {
289301
stat_ss_.clear();
302+
if (n_req_ == 0) {
303+
stat_ss_ << setprecision(4) << fixed << "dat: " << reader_->trace_path
304+
<< "\n"
305+
<< "number of requests: 0, number of objects: 0\n"
306+
<< "number of req GiB: 0.0000, number of obj GiB: 0.0000\n"
307+
<< "compulsory miss ratio (req/byte): 0.0000/0.0000\n"
308+
<< "object size weighted by req/obj: 0/0\n"
309+
<< "frequency mean: 0.0000\n"
310+
<< "time span: 0(0.0000 day)\n"
311+
<< "write: 0(0.0000), overwrite: 0(0.0000), del:0(0.0000)\n"
312+
<< "X-hit (number of obj accessed X times): ";
313+
for (int i = 0; i < track_n_hit_; i++) {
314+
stat_ss_ << "0(0.0000), ";
315+
}
316+
stat_ss_ << "\n";
317+
318+
stat_ss_ << "freq (fraction) of the most popular obj: ";
319+
for (int i = 0; i < track_n_popular_; i++) {
320+
stat_ss_ << "0(0.0000), ";
321+
}
322+
stat_ss_ << "\n";
323+
return stat_ss_.str();
324+
}
325+
290326
double cold_miss_ratio = (double)obj_map_.size() / (double)n_req_;
291327
double byte_cold_miss_ratio =
292328
(double)sum_obj_size_obj / (double)sum_obj_size_req;
@@ -354,7 +390,7 @@ void traceAnalyzer::TraceAnalyzer::post_processing() {
354390
}
355391
}
356392

357-
if (option_.popularity) {
393+
if (option_.popularity && !obj_map_.empty()) {
358394
popularity_stat_ = new Popularity(obj_map_);
359395
auto &sorted_freq = popularity_stat_->get_sorted_freq();
360396
int n = std::min(track_n_popular_, (int)sorted_freq.size());

‎libCacheSim/traceReader/reader.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,8 @@ int read_one_req(reader_t *const reader, request_t *const req) {
241241
return 1;
242242
}
243243

244-
if (reader->cap_at_n_req > 1 && reader->n_read_req >= reader->cap_at_n_req) {
244+
if (reader->cap_at_n_req >= 0 &&
245+
reader->n_read_req >= reader->cap_at_n_req) {
245246
DEBUG("read_one_req: processed %ld requests capped by the user\n",
246247
(long)reader->n_read_req);
247248
req->valid = false;

‎scripts/run_traceAnalyzer.sh‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
22

3+
set -euo pipefail
4+
35
function help() {
46
echo "Usage: $0 <tracepath> <trace_format> [<trace_format_parameters>]"
57
}
@@ -9,20 +11,33 @@ if [ $# -lt 2 ]; then
911
exit 1;
1012
fi
1113

12-
CURR_DIR=$(cd $(dirname $0); pwd)
14+
CURR_DIR=$(cd "$(dirname "$0")"; pwd)
15+
REPO_DIR=$(cd "${CURR_DIR}/.."; pwd)
16+
TRACE_ANALYZER="${REPO_DIR}/_build/bin/traceAnalyzer"
17+
18+
if [ ! -x "${TRACE_ANALYZER}" ]; then
19+
echo "traceAnalyzer binary not found at ${TRACE_ANALYZER}; build the project first" >&2
20+
exit 1
21+
fi
22+
1323
tracepath=$1
1424
trace_format=$2
15-
trace_format_parameters=${@:3}
25+
shift 2
26+
27+
analyzer_args=("${tracepath}" "${trace_format}" "--common")
28+
if [ $# -gt 0 ]; then
29+
analyzer_args+=("--trace-type-params" "$*")
30+
fi
1631

17-
./_build/bin/traceAnalyzer ${tracepath} ${trace_format} ${trace_format_parameters} --common
32+
"${TRACE_ANALYZER}" "${analyzer_args[@]}"
1833

19-
dataname=$(basename ${tracepath})
20-
python3 ${CURR_DIR}/traceAnalysis/access_pattern.py ${dataname}.accessRtime
21-
python3 ${CURR_DIR}/traceAnalysis/access_pattern.py ${dataname}.accessVtime
22-
python3 ${CURR_DIR}/traceAnalysis/req_rate.py ${dataname}.reqRate_w300
23-
python3 ${CURR_DIR}/traceAnalysis/size.py ${dataname}.size
24-
python3 ${CURR_DIR}/traceAnalysis/reuse.py ${dataname}.reuse
25-
python3 ${CURR_DIR}/traceAnalysis/popularity.py ${dataname}.popularity
34+
dataname=$(basename "${tracepath}")
35+
python3 "${CURR_DIR}/traceAnalysis/access_pattern.py" "${dataname}.accessRtime"
36+
python3 "${CURR_DIR}/traceAnalysis/access_pattern.py" "${dataname}.accessVtime"
37+
python3 "${CURR_DIR}/traceAnalysis/req_rate.py" "${dataname}.reqRate_w300"
38+
python3 "${CURR_DIR}/traceAnalysis/size.py" "${dataname}.size"
39+
python3 "${CURR_DIR}/traceAnalysis/reuse.py" "${dataname}.reuse"
40+
python3 "${CURR_DIR}/traceAnalysis/popularity.py" "${dataname}.popularity"
2641
# python3 ${CURR_DIR}/traceAnalysis/requestAge.py ${dataname}.requestAge
2742
# python3 ${CURR_DIR}/traceAnalysis/size_heatmap.py ${dataname}.sizeWindow_w300
2843
# python3 ${CURR_DIR}/traceAnalysis/futureReuse.py ${dataname}.access

0 commit comments

Comments
 (0)