WIP fuse filter

Author: rustaceanrob

src/blockfilter.cpp

@@ -2,8 +2,14 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <cassert>
+#include <cmath>
+#include <csignal>
+#include <cstdint>
 #include <mutex>
+#include <queue>
 #include <set>
+#include <stack>
 #include <string_view>
 
 #include <blockfilter.h>
@@ -21,6 +27,7 @@ using util::Join;
 
 static const std::map<BlockFilterType, std::string> g_filter_types = {
     {BlockFilterType::BASIC, "basic"},
+    {BlockFilterType::FUSE, "fuse"},
 };
 
 uint64_t GCSFilter::HashToRange(const Element& element) const
@@ -145,6 +152,115 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const
     return MatchInternal(queries.data(), queries.size());
 }
 
+uint64_t BinaryFuseFilter::Hash(const BinaryFuseFilter::Element& element) const
+{
+    return CSipHasher(m_siphash_k0, m_siphash_k1)
+        .Write(element)
+        .Finalize();
+}
+
+uint64_t BinaryFuseFilter::Mix(uint64_t key) const {
+    return CSipHasher(m_siphash_k0, m_siphash_k1)
+        .Write(key)
+        .Finalize();
+}
+
+uint16_t BinaryFuseFilter::Fingerprint(uint64_t key) const
+{
+    return static_cast<uint16_t>(key) ^ key >> 48;
+}
+
+std::tuple<uint32_t, uint32_t, uint32_t> BinaryFuseFilter::Slots(const uint64_t key) const
+{
+    uint32_t start_seg = key % (m_num_segments - m_arity + 1);
+    const auto h0_entropy = Mix(key);
+    const auto h1_entropy = Mix(h0_entropy);
+    const auto h2_entropy = Mix(h1_entropy);
+    uint32_t h0 = static_cast<uint32_t>((start_seg + 0) * m_segment_len + FastRange32(h0_entropy, m_segment_len));
+    uint32_t h1 = static_cast<uint32_t>((start_seg + 1) * m_segment_len + FastRange32(h1_entropy, m_segment_len));
+    uint32_t h2 = static_cast<uint32_t>((start_seg + 2) * m_segment_len + FastRange32(h2_entropy, m_segment_len));
+    return {h0, h1, h2};
+}
+
+bool BinaryFuseFilter::Query(const Element& element) const
+{
+    const auto key = Hash(element);
+    const auto [h0, h1, h2] = Slots(key);
+    const auto f = Fingerprint(key);
+    return f == (m_fingerprints[h0] ^ m_fingerprints[h1] ^ m_fingerprints[h2]);
+}
+
+BinaryFuseFilter::BinaryFuseFilter(const ElementSet& elements, const uint256& hash) {
+    m_siphash_k0 = hash.GetUint64(0);
+    m_siphash_k1 = hash.GetUint64(1);
+    const uint32_t n = static_cast<uint32_t>(elements.size());
+    const auto exp = static_cast<uint32_t>(std::floor((std::log(static_cast<double>(n)) / std::log(3.33) + 2.25)));
+    m_segment_len = std::max(uint32_t{2}, uint32_t{1} << exp);
+    auto target_array_len = static_cast<uint32_t>(std::ceil(n * 1.125));
+    m_num_segments = std::max(CeilDiv(target_array_len, m_segment_len), m_arity);
+    auto array_len = m_num_segments * m_segment_len;
+    m_fingerprints.assign(array_len, 0);
+    std::vector degrees = std::vector<Degree>(array_len);
+    degrees.assign(array_len, Degree());
+    for (const auto& element : elements) {
+        const auto key = Hash(element);
+        auto [h0, h1, h2] = Slots(key);
+        degrees[h0].m_degree++; degrees[h0].m_xor ^= key;
+        degrees[h1].m_degree++; degrees[h1].m_xor ^= key;
+        degrees[h2].m_degree++; degrees[h2].m_xor ^= key;
+    }
+    std::queue q = std::queue<uint32_t>{};
+    for (uint32_t i{0}; i < array_len; ++i) {
+        if (degrees[i].m_degree == 1) {
+            q.push(i);
+        }
+    }
+    std::stack p = std::stack<Assignment>{};
+    while (!q.empty()) {
+        const auto index = q.front();
+        q.pop();
+        if (degrees[index].m_degree != 1) continue;
+        uint64_t hash = degrees[index].m_xor;
+        p.emplace(index, hash);
+        const auto [h0, h1, h2] = Slots(hash);
+        degrees[h0].m_degree--;
+        degrees[h1].m_degree--;
+        degrees[h2].m_degree--;
+        degrees[h0].m_xor ^= hash;
+        degrees[h1].m_xor ^= hash;
+        degrees[h2].m_xor ^= hash;
+        if (degrees[h0].m_degree == 1) q.push(h0);
+        if (degrees[h1].m_degree == 1) q.push(h1);
+        if (degrees[h2].m_degree == 1) q.push(h2);
+    }
+    // check P is size N
+    // assert(p.size() == n);
+    while (!p.empty()) {
+        const auto assignment = p.top();
+        p.pop();
+        const auto hash = assignment.m_hash;
+        const auto f = Fingerprint(hash);
+        const auto [h0, h1, h2] = Slots(hash);
+        const auto i = assignment.m_index;
+        m_fingerprints[i] = f ^ m_fingerprints[h0] ^ m_fingerprints[h1] ^ m_fingerprints[h2];
+    }
+    DataStream writer{m_encoded};
+    this->Serialize(writer);
+}
+
+bool BinaryFuseFilter::MatchAny(const ElementSet& elements) const {
+    for (const auto& element : elements) {
+        if (Match(element)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool BinaryFuseFilter::Match(const Element& element) const {
+    return Query(element);
+}
+
 const std::string& BlockFilterTypeName(BlockFilterType filter_type)
 {
     static std::string unknown_retval;
@@ -226,7 +342,12 @@ BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const
     if (!BuildParams(params)) {
         throw std::invalid_argument("unknown filter_type");
     }
-    m_filter = std::make_unique<GCSFilter>(params, BasicFilterElements(block, block_undo));
+    if (filter_type == BlockFilterType::BASIC) {
+        m_filter = std::make_unique<GCSFilter>(params, BasicFilterElements(block, block_undo));
+    }
+    if (filter_type == BlockFilterType::FUSE) {
+        m_filter = std::make_unique<BinaryFuseFilter>(BinaryFuseFilter::build(BasicFilterElements(block, block_undo), m_block_hash));
+    }
 }
 
 bool BlockFilter::BuildParams(GCSFilter::Params& params) const
@@ -238,6 +359,7 @@ bool BlockFilter::BuildParams(GCSFilter::Params& params) const
         params.m_P = BASIC_FILTER_P;
         params.m_M = BASIC_FILTER_M;
         return true;
+    case BlockFilterType::FUSE: return true;
     case BlockFilterType::INVALID:
         return false;
     }

src/blockfilter.h

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include <attributes.h>
+#include <serialize.h>
 #include <uint256.h>
 #include <util/bytevectorhash.h>
 
@@ -106,12 +107,64 @@ class GCSFilter : public BlockFilterBase
     bool MatchAny(const ElementSet& elements) const override;
 };
 
+class BinaryFuseFilter : public BlockFilterBase
+{
+private:
+    struct Degree
+    {
+        uint32_t m_degree = 0;
+        uint64_t m_xor = 0;
+    };
+
+    struct Assignment
+    {
+        uint32_t m_index = 0;
+        uint64_t m_hash = 0;
+    };
+
+    static constexpr uint32_t m_arity{3};
+    static constexpr double m_load_factor{1.125};
+    uint64_t m_siphash_k0;
+    uint64_t m_siphash_k1;
+    uint32_t m_num_segments;
+    uint32_t m_segment_len;
+    std::vector<uint16_t> m_fingerprints;
+    std::vector<unsigned char> m_encoded;
+
+    uint64_t Hash(const Element& element) const;
+
+    uint64_t Mix(uint64_t key) const;
+
+    uint16_t Fingerprint(uint64_t key) const;
+
+    std::tuple<uint32_t, uint32_t, uint32_t> Slots(uint64_t key) const;
+
+    bool Query(const Element& element) const;
+
+    BinaryFuseFilter(const ElementSet& elements, const uint256& hash);
+
+public:
+    static BinaryFuseFilter build(const ElementSet& elements, const uint256& hash) {
+        return BinaryFuseFilter{elements, hash};
+    }
+
+    SERIALIZE_METHODS(BinaryFuseFilter, obj) { READWRITE(obj.m_fingerprints); }
+
+    const std::vector<unsigned char>& GetEncoded() const LIFETIMEBOUND override { return m_encoded; };
+
+    bool Match(const Element& element) const override;
+
+    bool MatchAny(const ElementSet& elements) const override;
+
+};
+
 constexpr uint8_t BASIC_FILTER_P = 19;
 constexpr uint32_t BASIC_FILTER_M = 784931;
 
 enum class BlockFilterType : uint8_t
 {
     BASIC = 0,
+    FUSE = 1,
     INVALID = 255,
 };
 

src/init.cpp

@@ -995,7 +995,7 @@ bool AppInitParameterInteraction(const ArgsManager& args)
 
     // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled.
     if (args.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) {
-        if (!g_enabled_filter_types.contains(BlockFilterType::BASIC)) {
+        if (!g_enabled_filter_types.contains(BlockFilterType::BASIC) && !g_enabled_filter_types.contains(BlockFilterType::FUSE)) {
             return InitError(_("Cannot set -peerblockfilters without -blockfilterindex."));
         }
 

src/net_processing.cpp

@@ -3267,7 +3267,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer,
                                                 BlockFilterIndex*& filter_index)
 {
     const bool supported_filter_type =
-        (filter_type == BlockFilterType::BASIC &&
+        ((filter_type == BlockFilterType::BASIC || filter_type == BlockFilterType::FUSE) &&
          (peer.m_our_services & NODE_COMPACT_FILTERS));
     if (!supported_filter_type) {
         LogDebug(BCLog::NET, "peer requested unsupported block filter type: %d, %s",

src/test/blockfilter_tests.cpp

@@ -41,6 +41,29 @@ BOOST_AUTO_TEST_CASE(gcsfilter_test)
     }
 }
 
+BOOST_AUTO_TEST_CASE(binaryfusefilter_test)
+{
+    BinaryFuseFilter::ElementSet included_elements, excluded_elements;
+    for (int i = 0; i < 100; ++i) {
+        BinaryFuseFilter::Element element1(32);
+        element1[0] = i;
+        included_elements.insert(std::move(element1));
+
+        BinaryFuseFilter::Element element2(32);
+        element2[1] = i;
+        excluded_elements.insert(std::move(element2));
+    }
+
+    const BinaryFuseFilter filter = BinaryFuseFilter::build(included_elements, uint256{1 < 100});
+    for (const auto& element : included_elements) {
+        BOOST_CHECK(filter.Match(element));
+
+        auto insertion = excluded_elements.insert(element);
+        BOOST_CHECK(filter.MatchAny(excluded_elements));
+        excluded_elements.erase(insertion.first);
+    }
+}
+
 BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor)
 {
     GCSFilter filter;
@@ -109,6 +132,16 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
         BOOST_CHECK(!filter.Match(GCSFilter::Element(script.begin(), script.end())));
     }
 
+    BlockFilter block_filter_2(BlockFilterType::FUSE, block, block_undo);
+    const BlockFilterBase& filter_2 = block_filter_2.GetFilter();
+
+    for (const CScript& script : included_scripts) {
+        BOOST_CHECK(filter_2.Match(BlockFilterBase::Element(script.begin(), script.end())));
+    }
+    for (const CScript& script : excluded_scripts) {
+        BOOST_CHECK(!filter_2.Match(BlockFilterBase::Element(script.begin(), script.end())));
+    }
+
     // Test serialization/unserialization.
     BlockFilter block_filter2;