refactoring of encoders

Author: jermp

README.md

@@ -100,16 +100,6 @@ For a testing environment, use the following instead:
 
 (NOTE: Beware that the software will result in a much slower execution when running in debug mode and using sanitizers. Use this only for debug purposes, not to run performance tests.)
 
-### Enable All Encoders
-
-By default, you can choose between three encoders to compress the PTHash
-data structure (see the output of `./build --help` for suggestions).
-
-If you want to test all the encoders we tested in the papers,
-compile again with
-
-	cmake .. -D PTHASH_ENABLE_ALL_ENCODERS=On
-
 ### Enable Large Bucket-Id Type
 
 By default, PTHash assumes there are less than $2^{32}$ buckets, hence 32-bit integers are used
@@ -233,10 +223,6 @@ Giulio on 13/04/2025: Update this section with new benchmarks.
 
 <!-- The script `script/run_benchmark.sh` runs some trade-off configurations (encoder, $\alpha$, $\lambda$) that have been tested in the papers, on 100M and 1000M keys.
 
-Be sure you run the benchmark after compiling with
-
-	cmake .. -D PTHASH_ENABLE_ALL_ENCODERS=On
-
 From within the directory where the code has been compiled, just run
 
 	bash ../script/run_benchmark.sh 100000000 2> results.json

include/dense_partitioned_phf.hpp

@@ -166,6 +166,6 @@ struct dense_partitioned_phf  //
 };
 
 template <typename Hasher>
-using phobic = dense_partitioned_phf<Hasher, opt_bucketer, inter_C_inter_R, true>;
+using phobic = dense_partitioned_phf<Hasher, opt_bucketer, R_int, true>;
 
 }  // namespace pthash

include/utils/dense_encoders.hpp

@@ -24,7 +24,7 @@ struct dense_mono : dense_encoder {
     }
 
     static std::string name() {
-        return "mono-" + Encoder::name();
+        return Encoder::name();
     }
 
     size_t size() const {
@@ -100,7 +100,7 @@ struct dense_interleaved : dense_encoder {
     }
 
     static std::string name() {
-        return "inter-" + Encoder::name();
+        return Encoder::name() + "-int";
     }
 
     inline uint64_t access(const uint64_t partition, const uint64_t bucket) const {
@@ -109,7 +109,7 @@ struct dense_interleaved : dense_encoder {
     }
 
     uint64_t num_bits() const {
-        uint64_t sum = 0;
+        uint64_t sum = 8 * sizeof(uint64_t);  // for std::vector size
         for (auto const& e : m_encoders) sum += e.num_bits();
         return sum;
     }
@@ -133,93 +133,13 @@ struct dense_interleaved : dense_encoder {
     std::vector<Encoder> m_encoders;
 };
 
-template <typename Front, typename Back, uint64_t numerator = 1, uint64_t denominator = 3>
-struct dense_dual : dense_encoder {
-    template <typename Iterator>
-    void encode(Iterator begin,                                                        //
-                const uint64_t num_partitions,                                         //
-                const uint64_t num_buckets_per_partition, const uint64_t num_threads)  //
-    {
-        m_front_size = num_buckets_per_partition * (static_cast<double>(numerator) / denominator);
-        if (num_threads == 1) {
-            if (m_front_size != 0) m_front.encode(begin, num_partitions, m_front_size, 1);
-            if (num_buckets_per_partition - m_front_size != 0)
-                m_back.encode(begin + m_front_size * num_partitions, num_partitions,
-                              num_buckets_per_partition - m_front_size, 1);
-        } else {
-            uint64_t m_front_threads =
-                (num_threads * m_front_size + num_buckets_per_partition - 1) /
-                num_buckets_per_partition;
-            auto exe = [&]() {
-                if (m_front_size != 0)
-                    m_front.encode(begin, num_partitions, m_front_size, m_front_threads);
-            };
-            std::thread frontThread = std::thread(exe);
-            if (num_buckets_per_partition - m_front_size != 0)
-                m_back.encode(begin + m_front_size * num_partitions, num_partitions,
-                              num_buckets_per_partition - m_front_size,
-                              num_threads - m_front_threads);
-            if (frontThread.joinable()) frontThread.join();
-        }
-    }
-
-    static std::string name() {
-        std::ostringstream oss;
-        oss << Front::name() << "-" << Back::name() << "-" << std::fixed << std::setprecision(2)
-            << static_cast<double>(numerator) / denominator;
-        return oss.str();
-    }
-
-    size_t num_bits() const {
-        return sizeof(m_front_size) * 8 + m_front.num_bits() + m_back.num_bits();
-    }
-
-    uint64_t access(uint64_t i) const {
-        if (i < m_front.size()) return m_front.access(i);
-        return m_back.access(i - m_front.size());
-    }
-
-    inline uint64_t access(const uint64_t partition, const uint64_t bucket) const {
-        if (bucket < m_front_size) return m_front.access(partition, bucket);
-        return m_back.access(partition, bucket - m_front_size);
-    }
-
-    template <typename Visitor>
-    void visit(Visitor& visitor) const {
-        visit_impl(visitor, *this);
-    }
-
-    template <typename Visitor>
-    void visit(Visitor& visitor) {
-        visit_impl(visitor, *this);
-    }
-
-private:
-    template <typename Visitor, typename T>
-    static void visit_impl(Visitor& visitor, T&& t) {
-        visitor.visit(t.m_front_size);
-        visitor.visit(t.m_front);
-        visitor.visit(t.m_back);
-    }
-
-    uint64_t m_front_size;
-    Front m_front;
-    Back m_back;
-};
+typedef dense_mono<compact> C_mono;
+typedef dense_mono<dictionary> D_mono;
+typedef dense_mono<rice> R_mono;
+typedef dense_mono<elias_fano> EF_mono;
 
-typedef dense_mono<rice> mono_R;
-typedef dense_interleaved<rice> inter_R;
-typedef dense_mono<compact> mono_C;
-typedef dense_interleaved<compact> inter_C;
-typedef dense_mono<dictionary> mono_D;
-typedef dense_interleaved<dictionary> inter_D;
-typedef dense_mono<elias_fano> mono_EF;
-typedef dense_interleaved<elias_fano> inter_EF;
-
-/* dual_interleaved encoders */
-typedef dense_dual<mono_C, mono_R, 1, 3> mono_C_mono_R;
-typedef dense_dual<inter_C, inter_R, 1, 3> inter_C_inter_R;
-typedef dense_dual<mono_D, mono_R, 1, 3> mono_D_mono_R;
-typedef dense_dual<inter_D, inter_R, 1, 3> inter_D_inter_R;
+typedef dense_interleaved<compact> C_int;
+typedef dense_interleaved<dictionary> D_int;
+typedef dense_interleaved<rice> R_int;
 
 }  // namespace pthash

script/run_benchmark.py

@@ -54,28 +54,29 @@ def run_build(n, base_filename=None):
 
     for l in lambda_values:
         for a in alpha_values:
-
-            cmd = [
-                "./build",
-                "-n", str(n),
-                "-l", f"{l:.2f}",
-                "-a", f"{a:.2f}",
-                "-e", "all",
-                "-b", "skew",
-                "-s", "0",
-                "-q", str(n),
-                "-t", str(num_threads),
-                "--minimal",
-                "--check",
-                "--verbose"
-            ]
-
-            run_cmd("SINGLE", l, a, cmd, log_file, results_file)
-
-            avg_partition_size = n / (num_threads * num_partitions_per_thread)
-            run_cmd("PARTITIONED", l, a, cmd + ["-p", str(avg_partition_size)], log_file, results_file)
-
-            run_cmd("DENSE-PARTITIONED", l, a, cmd + ["--dense"], log_file, results_file)
+            for b in ["skew", "opt"]:
+                cmd = [
+                    "./build",
+                    "-n", str(n),
+                    "-l", str(l),
+                    "-a", str(a),
+                    "-e", "all",
+                    "-b", b,
+                    "-s", "0",
+                    "-q", str(n),
+                    "-t", str(num_threads),
+                    "--minimal",
+                    "--check",
+                    "--verbose"
+                ]
+
+                run_cmd("SINGLE", l, a, cmd, log_file, results_file)
+
+                avg_partition_size = n / (num_threads * num_partitions_per_thread)
+                run_cmd("PARTITIONED", l, a, cmd + ["-p", str(avg_partition_size)], log_file, results_file)
+
+                if a == 1.0:
+                    run_cmd("DENSE-PARTITIONED", l, a, cmd + ["--dense"], log_file, results_file)
 
     log_file.close()
     results_file.close()