refactor: Improve Zipf fitting calculation in popularity analysis

* Updated the calculation of Zipf alpha to use a more robust linear regression method.
* Enhanced logging for regression failures with specific error messages.
* Cleaned up the code for calculating log frequencies and ranks for better readability.

Author: 1a1a11a

doc/quickstart_traceAnalyzer.md

@@ -62,7 +62,7 @@ The trace analyzer will generate statistics of the trace and save them to `stat`
     write: 0(0), overwrite: 0(0), del:0(0)
     request rate min 1753.7533 req/s, max 1986.3433 req/s, window 300s
     object rate min 300.3567 obj/s, max 319.8633 obj/s, window 300s
-    popularity: Zipf linear fitting slope=0.9472
+    popularity: Zipf alpha=0.9472, R2=0.97
     X-hit (number of obj accessed X times): 323699(0.3606), 218436(0.2433), 51516(0.0574), 128181(0.1428), 48785(0.0543), 25172(0.0280), 14606(0.0163), 14769(0.0165), 
     freq (fraction) of the most popular obj: 546563(0.0547), 365140(0.0365), 221311(0.0221), 190811(0.0191), 154037(0.0154), 151832(0.0152), 127070(0.0127), 98851(0.0099), 
 </details>

libCacheSim/traceAnalyzer/popularity.cpp

@@ -66,21 +66,28 @@ void Popularity::run(obj_info_map_type &obj_map) {
     WARN("%s\n", fit_fail_reason_.c_str());
   }
 
-  /* calculate Zipf alpha using linear regression */
-  vector<double> log_freq(obj_map.size());
-  vector<double> log_rank(obj_map.size());
+  /* calculate Zipf alpha using linear regression: log(freq) = -alpha*log(rank) + c */
+  const size_t n = freq_vec_.size();
+  vector<double> log_freq(n);
+  vector<double> log_rank(n);
 
-  int i = 0;
-  for_each(log_freq.begin(), log_freq.end(),
-           [&](double &item) { item = log(freq_vec_[i++]); });
-  i = 0;
-  for_each(log_rank.begin(), log_rank.end(), [&](double &item) {
-    ++i;
-    item = log(i);
-  });
+  for (size_t i = 0; i < n; i++) {
+    log_freq[i] = log(static_cast<double>(freq_vec_[i]));
+    log_rank[i] = log(static_cast<double>(i + 1));
+  }
 
-  /* TODO: a better linear regression with intercept and R2 */
-  slope_ = -PopularityUtils::slope(log_rank, log_freq);
+  double reg_slope, reg_intercept, r;
+  int err = linreg(static_cast<int>(n), log_rank.data(), log_freq.data(),
+                  &reg_slope, &reg_intercept, &r);
+  if (err != 0) {
+    fit_fail_reason_ = "popularity: singular regression matrix (e.g. uniform)";
+    WARN("%s\n", fit_fail_reason_.c_str());
+    return;
+  }
+  /* Zipf: log(freq) = -alpha*log(rank) + c, so reg_slope = -alpha */
+  slope_ = -reg_slope;
+  intercept_ = reg_intercept;
+  r2_ = (std::isfinite(r) ? r * r : 0.0);
 
   has_run = true;
 }

libCacheSim/traceAnalyzer/popularity.h

@@ -47,8 +47,8 @@ class Popularity {
       os << popularity.fit_fail_reason_ << "\n";
     else
       os << std::setprecision(4)
-         << "popularity: Zipf linear fitting slope=" << popularity.slope_
-         << ", intercept=" << popularity.intercept_ << ", R2=" << popularity.r2_
+         << "popularity: Zipf alpha=" << popularity.slope_
+         << ", R2=" << popularity.r2_
          << "\n";
 
     return os;

scripts/traceAnalysis/popularity.py

@@ -80,17 +80,14 @@ def plot_popularity_Zipf(datapath, figname_prefix=""):
 
     x = np.log(np.arange(1, 1 + len(sorted_freq)))
     y = np.log(np.array(sorted_freq))
-    slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
+    reg_slope, _, r_value, p_value, std_err = stats.linregress(x, y)
+    alpha = -reg_slope
+    r2 = r_value * r_value
 
     if sorted_freq[0] < 100:
-        s = "{:48} {:12} obj alpha 0, r^2 0 (the most popular object has less than 100 requests)".format(
-            figname_prefix,
-            len(sorted_freq),
-        )
+        s = "popularity: Zipf alpha=0, R2=0 (the most popular object has less than 100 requests)"
     else:
-        s = "{:48} {:12} obj alpha {:.4f}, r^2 {:.4f}".format(
-            figname_prefix, len(sorted_freq), -slope, r_value * r_value
-        )
+        s = "popularity: Zipf alpha={:.4f}, R2={:.4f}".format(alpha, r2)
 
     logger.info(s)