Ranked OR TAAT

CMakeLists.txt

@@ -40,7 +40,6 @@ list(APPEND LCOV_REMOVE_PATTERNS "'${PROJECT_SOURCE_DIR}/external/*'")
 
 
 if (UNIX)
-
    # For hardware popcount and other special instructions
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
 
@@ -62,8 +61,6 @@ endif()
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-link_libraries(Threads::Threads)
-
 
 include_directories(include)
 add_library(pisa INTERFACE)

include/pisa/accumulator/simple_accumulator.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+namespace pisa {
+
+struct Simple_Accumulator : public std::vector<float> {
+    Simple_Accumulator(std::ptrdiff_t size) : std::vector<float>(size) {}
+    void init() { std::fill(begin(), end(), 0.0); }
+    void accumulate(uint32_t doc, float score) { operator[](doc) += score; }
+    void aggregate(topk_queue &topk) {
+        uint64_t docid = 0u;
+        std::for_each(begin(), end(), [&](auto score) { topk.insert(score, docid++); });
+    }
+};
+
+}

include/pisa/block_posting_list.hpp

@@ -82,6 +82,7 @@ namespace pisa {
 
         class document_enumerator {
         public:
+
             document_enumerator(uint8_t const* data, uint64_t universe,
                                 size_t term_id = 0)
                 : m_n(0) // just to silence warnings

include/pisa/freq_index.hpp

@@ -76,6 +76,7 @@ namespace pisa {
 
         class document_enumerator {
         public:
+
             void reset()
             {
                 m_cur_pos = 0;

include/pisa/query/algorithm/and_query.hpp

@@ -2,11 +2,12 @@
 
 namespace pisa {
 
-template <bool with_freqs>
+template <typename Index, bool with_freqs>
 struct and_query {
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec terms) const {
+    and_query(Index const &index) : m_index(index) {}
+
+    uint64_t operator()(term_id_vec terms) const {
         if (terms.empty())
             return 0;
         remove_duplicate_terms(terms);
@@ -16,7 +17,7 @@ struct and_query {
         enums.reserve(terms.size());
 
         for (auto term : terms) {
-            enums.push_back(index[term]);
+            enums.push_back(m_index[term]);
         }
 
         // sort by increasing frequency
@@ -27,7 +28,7 @@ struct and_query {
         uint64_t results   = 0;
         uint64_t candidate = enums[0].docid();
         size_t   i         = 1;
-        while (candidate < index.num_docs()) {
+        while (candidate < m_index.num_docs()) {
             for (; i < enums.size(); ++i) {
                 enums[i].next_geq(candidate);
                 if (enums[i].docid() != candidate) {
@@ -52,6 +53,9 @@ struct and_query {
         }
         return results;
     }
+
+   private:
+    Index const &m_index;
 };
 
 } // namespace pisa

include/pisa/query/algorithm/block_max_maxscore_query.hpp

@@ -2,22 +2,22 @@
 
 namespace pisa {
 
-template <typename WandType>
+template <typename Index, typename WandType>
 struct block_max_maxscore_query {
 
     typedef bm25 scorer_type;
 
-    block_max_maxscore_query(WandType const &wdata, uint64_t k) : m_wdata(&wdata), m_topk(k) {}
+    block_max_maxscore_query(Index const &index, WandType const &wdata, uint64_t k)
+        : m_index(index), m_wdata(&wdata), m_topk(k) {}
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec const &terms) {
+    uint64_t operator()(term_id_vec const &terms) {
         m_topk.clear();
         if (terms.empty())
             return 0;
 
         auto query_term_freqs = query_freqs(terms);
 
-        uint64_t                                        num_docs = index.num_docs();
+        uint64_t                                        num_docs = m_index.num_docs();
         typedef typename Index::document_enumerator     enum_type;
         typedef typename WandType::wand_data_enumerator wdata_enum;
 
@@ -32,7 +32,7 @@ struct block_max_maxscore_query {
         enums.reserve(query_term_freqs.size());
 
         for (auto term : query_term_freqs) {
-            auto list       = index[term.first];
+            auto list       = m_index[term.first];
             auto w_enum     = m_wdata->getenum(term.first);
             auto q_weight   = scorer_type::query_term_weight(term.second, list.size(), num_docs);
             auto max_weight = q_weight * m_wdata->max_term_weight(term.first);
@@ -66,10 +66,10 @@ struct block_max_maxscore_query {
                              })
                 ->docs_enum.docid();
 
-        while (non_essential_lists < ordered_enums.size() && cur_doc < index.num_docs()) {
+        while (non_essential_lists < ordered_enums.size() && cur_doc < m_index.num_docs()) {
             float    score    = 0;
             float    norm_len = m_wdata->norm_len(cur_doc);
-            uint64_t next_doc = index.num_docs();
+            uint64_t next_doc = m_index.num_docs();
             for (size_t i = non_essential_lists; i < ordered_enums.size(); ++i) {
                 if (ordered_enums[i]->docs_enum.docid() == cur_doc) {
                     score +=
@@ -129,6 +129,7 @@ struct block_max_maxscore_query {
     std::vector<std::pair<float, uint64_t>> const &topk() const { return m_topk.topk(); }
 
    private:
+    Index const &   m_index;
     WandType const *m_wdata;
     topk_queue      m_topk;
 };

include/pisa/query/algorithm/block_max_wand_query.hpp

@@ -2,20 +2,20 @@
 
 namespace pisa {
 
-template <typename WandType>
+template <typename Index, typename WandType>
 struct block_max_wand_query {
     typedef bm25 scorer_type;
 
-    block_max_wand_query(WandType const &wdata, uint64_t k) : m_wdata(&wdata), m_topk(k) {}
+    block_max_wand_query(Index const &index, WandType const &wdata, uint64_t k)
+        : m_index(index), m_wdata(&wdata), m_topk(k) {}
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec const &terms) {
+    uint64_t operator()(term_id_vec const &terms) {
         m_topk.clear();
 
         if (terms.empty())
             return 0;
         auto                                            query_term_freqs = query_freqs(terms);
-        uint64_t                                        num_docs         = index.num_docs();
+        uint64_t                                        num_docs         = m_index.num_docs();
         typedef typename Index::document_enumerator     enum_type;
         typedef typename WandType::wand_data_enumerator wdata_enum;
 
@@ -30,7 +30,7 @@ struct block_max_wand_query {
         enums.reserve(query_term_freqs.size());
 
         for (auto term : query_term_freqs) {
-            auto list     = index[term.first];
+            auto list     = m_index[term.first];
             auto w_enum   = m_wdata->getenum(term.first);
             auto q_weight = scorer_type::query_term_weight(term.second, list.size(), num_docs);
 
@@ -204,6 +204,7 @@ struct block_max_wand_query {
     topk_queue const &get_topk() const { return m_topk; }
 
    private:
+    Index const &   m_index;
     WandType const *m_wdata;
     topk_queue      m_topk;
 };

include/pisa/query/algorithm/maxscore_query.hpp

@@ -2,22 +2,21 @@
 
 namespace pisa {
 
-template <typename WandType>
+template <typename Index, typename WandType>
 struct maxscore_query {
 
     typedef bm25 scorer_type;
 
-    maxscore_query(WandType const &wdata, uint64_t k) : m_wdata(&wdata), m_topk(k) {}
+    maxscore_query(Index const &index, WandType const &wdata, uint64_t k) : m_index(index), m_wdata(&wdata), m_topk(k) {}
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec const &terms) {
+    uint64_t operator()(term_id_vec const &terms) {
         m_topk.clear();
         if (terms.empty())
             return 0;
 
         auto query_term_freqs = query_freqs(terms);
 
-        uint64_t                                    num_docs = index.num_docs();
+        uint64_t                                    num_docs = m_index.num_docs();
         typedef typename Index::document_enumerator enum_type;
         struct scored_enum {
             enum_type docs_enum;
@@ -29,7 +28,7 @@ struct maxscore_query {
         enums.reserve(query_term_freqs.size());
 
         for (auto term : query_term_freqs) {
-            auto list       = index[term.first];
+            auto list       = m_index[term.first];
             auto q_weight   = scorer_type::query_term_weight(term.second, list.size(), num_docs);
             auto max_weight = q_weight * m_wdata->max_term_weight(term.first);
             enums.push_back(scored_enum{std::move(list), q_weight, max_weight});
@@ -62,10 +61,10 @@ struct maxscore_query {
                              })
                 ->docs_enum.docid();
 
-        while (non_essential_lists < ordered_enums.size() && cur_doc < index.num_docs()) {
+        while (non_essential_lists < ordered_enums.size() && cur_doc < m_index.num_docs()) {
             float    score    = 0;
             float    norm_len = m_wdata->norm_len(cur_doc);
-            uint64_t next_doc = index.num_docs();
+            uint64_t next_doc = m_index.num_docs();
             for (size_t i = non_essential_lists; i < ordered_enums.size(); ++i) {
                 if (ordered_enums[i]->docs_enum.docid() == cur_doc) {
                     score +=
@@ -109,6 +108,7 @@ struct maxscore_query {
     std::vector<std::pair<float, uint64_t>> const &topk() const { return m_topk.topk(); }
 
    private:
+    Index const &   m_index;
     WandType const *m_wdata;
     topk_queue      m_topk;
 };

include/pisa/query/algorithm/or_query.hpp

@@ -2,11 +2,12 @@
 
 namespace pisa {
 
-template <bool with_freqs>
+template <typename Index, bool with_freqs>
 struct or_query {
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec terms) const {
+    or_query(Index const &index) : m_index(index) {}
+
+    uint64_t operator()(term_id_vec terms) const {
         if (terms.empty())
             return 0;
         remove_duplicate_terms(terms);
@@ -16,7 +17,7 @@ struct or_query {
         enums.reserve(terms.size());
 
         for (auto term : terms) {
-            enums.push_back(index[term]);
+            enums.push_back(m_index[term]);
         }
 
         uint64_t results = 0;
@@ -27,9 +28,9 @@ struct or_query {
                                             })
                                ->docid();
 
-        while (cur_doc < index.num_docs()) {
+        while (cur_doc < m_index.num_docs()) {
             results += 1;
-            uint64_t next_doc = index.num_docs();
+            uint64_t next_doc = m_index.num_docs();
             for (size_t i = 0; i < enums.size(); ++i) {
                 if (enums[i].docid() == cur_doc) {
                     if (with_freqs) {
@@ -47,6 +48,9 @@ struct or_query {
 
         return results;
     }
+
+   private:
+    Index const &m_index;
 };
 
 } // namespace pisa

include/pisa/query/algorithm/ranked_and_query.hpp

@@ -2,23 +2,23 @@
 
 namespace pisa {
 
-template <typename WandType>
+template <typename Index, typename WandType>
 struct ranked_and_query {
 
     typedef bm25 scorer_type;
 
-    ranked_and_query(WandType const &wdata, uint64_t k) : m_wdata(&wdata), m_topk(k) {}
+    ranked_and_query(Index const &index, WandType const &wdata, uint64_t k)
+        : m_index(index), m_wdata(&wdata), m_topk(k) {}
 
-    template <typename Index>
-    uint64_t operator()(Index const &index, term_id_vec terms) {
+    uint64_t operator()(term_id_vec terms) {
         size_t results = 0;
         m_topk.clear();
         if (terms.empty())
             return 0;
 
         auto query_term_freqs = query_freqs(terms);
 
-        uint64_t                                    num_docs = index.num_docs();
+        uint64_t                                    num_docs = m_index.num_docs();
         typedef typename Index::document_enumerator enum_type;
         struct scored_enum {
             enum_type docs_enum;
@@ -29,7 +29,7 @@ struct ranked_and_query {
         enums.reserve(query_term_freqs.size());
 
         for (auto term : query_term_freqs) {
-            auto list     = index[term.first];
+            auto list     = m_index[term.first];
             auto q_weight = scorer_type::query_term_weight(term.second, list.size(), num_docs);
             enums.push_back(scored_enum{std::move(list), q_weight});
         }
@@ -41,7 +41,7 @@ struct ranked_and_query {
 
         uint64_t candidate = enums[0].docs_enum.docid();
         size_t   i         = 1;
-        while (candidate < index.num_docs()) {
+        while (candidate < m_index.num_docs()) {
             for (; i < enums.size(); ++i) {
                 enums[i].docs_enum.next_geq(candidate);
                 if (enums[i].docs_enum.docid() != candidate) {
@@ -80,6 +80,7 @@ struct ranked_and_query {
     topk_queue &get_topk() { return m_topk; }
 
    private:
+    Index const &   m_index;
     WandType const *m_wdata;
     topk_queue      m_topk;
 };