|
| 1 | +// |
| 2 | +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance |
| 3 | +// with the License. You may obtain a copy of the License at |
| 4 | +// |
| 5 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 6 | +// |
| 7 | +// Unless required by applicable law or agreed to in writing, software distributed under the License |
| 8 | +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
| 9 | +// or implied. See the License for the specific language governing permissions and limitations under the License. |
| 10 | +#ifndef KNOWHERE_KNOWHERE_H |
| 11 | +#define KNOWHERE_KNOWHERE_H |
| 12 | +#include <cmath> |
| 13 | +#include <cstring> |
| 14 | +#include <fstream> |
| 15 | +#include <functional> |
| 16 | +#include <vector> |
| 17 | + |
| 18 | +#include "io/memory_io.h" |
| 19 | +#include "knowhere/bitsetview.h" |
| 20 | +#include "knowhere/utils.h" |
| 21 | +namespace knowhere { |
| 22 | +template <typename T> |
| 23 | +class BloomFilter { |
| 24 | + public: |
| 25 | + explicit BloomFilter(size_t expected_elements, double false_positive_prob) |
| 26 | + : n(expected_elements), p(false_positive_prob) { |
| 27 | + m = static_cast<size_t>(-(n * log(p)) / (log(2) * log(2))); |
| 28 | + k = static_cast<int>(m / n * log(2)); |
| 29 | + m = std::max<size_t>(m, 1); |
| 30 | + k = std::max(k, 1); |
| 31 | + bits.resize(m, false); |
| 32 | + } |
| 33 | + |
| 34 | + void |
| 35 | + add(const T& element) { |
| 36 | + size_t glb_hash = hash((const char*)&element, sizeof(element), 0); |
| 37 | + for (int i = 0; i < k; ++i) { |
| 38 | + size_t pos = (glb_hash + i) % m; |
| 39 | + bits[pos] = true; |
| 40 | + } |
| 41 | + } |
| 42 | + |
| 43 | + bool |
| 44 | + contains(const T& element) const { |
| 45 | + size_t glb_hash = hash((const char*)&element, sizeof(element), 0); |
| 46 | + for (int i = 0; i < k; ++i) { |
| 47 | + size_t pos = (glb_hash + i) % m; |
| 48 | + if (!bits[pos]) |
| 49 | + return false; |
| 50 | + } |
| 51 | + return true; |
| 52 | + } |
| 53 | + |
| 54 | + void |
| 55 | + save(MemoryIOWriter& writer) const { |
| 56 | + writeBinaryPOD(writer, m); |
| 57 | + writeBinaryPOD(writer, k); |
| 58 | + writeBinaryPOD(writer, n); |
| 59 | + writeBinaryPOD(writer, p); |
| 60 | + auto bytes_num = (m + 8 - 1) / 8; |
| 61 | + std::vector<char> buffer(bytes_num, 0); |
| 62 | + for (size_t i = 0; i < m; ++i) { |
| 63 | + if (bits[i]) { |
| 64 | + buffer[i / 8] |= (1 << (i % 8)); |
| 65 | + } |
| 66 | + } |
| 67 | + writer.write(buffer.data(), buffer.size()); |
| 68 | + } |
| 69 | + |
| 70 | + void |
| 71 | + load(MemoryIOReader& reader) { |
| 72 | + readBinaryPOD(reader, m); |
| 73 | + readBinaryPOD(reader, k); |
| 74 | + readBinaryPOD(reader, n); |
| 75 | + readBinaryPOD(reader, p); |
| 76 | + bits.clear(); |
| 77 | + bits.resize(m); |
| 78 | + auto bytes_num = (m + 8 - 1) / 8; |
| 79 | + std::vector<char> buffer(bytes_num); |
| 80 | + reader.read(buffer.data(), bytes_num); |
| 81 | + for (size_t i = 0; i < m; ++i) { |
| 82 | + bool bit = (buffer[i / 8] >> (i % 8)) & 1; |
| 83 | + bits.push_back(bit); |
| 84 | + } |
| 85 | + } |
| 86 | + size_t |
| 87 | + size() const { |
| 88 | + return n; |
| 89 | + } |
| 90 | + double |
| 91 | + false_positive_rate() const { |
| 92 | + return p; |
| 93 | + } |
| 94 | + size_t |
| 95 | + memory_usage() const { |
| 96 | + return m / 8; |
| 97 | + } |
| 98 | + |
| 99 | + private: |
| 100 | + static constexpr size_t multiplier = 31; |
| 101 | + std::vector<bool> bits; |
| 102 | + size_t n = 0; |
| 103 | + double p = 0.0; |
| 104 | + size_t m = 0; |
| 105 | + int k = 0; |
| 106 | + |
| 107 | + size_t |
| 108 | + hash(const char* data, size_t length, size_t bucket_i) const { |
| 109 | + if (data == nullptr) { |
| 110 | + throw std::runtime_error("can't hash null data."); |
| 111 | + } |
| 112 | + size_t result = 0; |
| 113 | + for (size_t i = 0; i < length; ++i) { |
| 114 | + result = (result * multiplier) + static_cast<size_t>(data[i]); |
| 115 | + } |
| 116 | + return (result + bucket_i) % m; |
| 117 | + } |
| 118 | +}; |
| 119 | +} // namespace knowhere |
| 120 | +#endif |
0 commit comments